s3db.js 11.0.2 → 11.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +612 -308
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +612 -308
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/concerns/plugin-storage.js +274 -9
- package/src/plugins/audit.plugin.js +94 -18
- package/src/plugins/eventual-consistency/analytics.js +131 -15
- package/src/plugins/eventual-consistency/config.js +3 -0
- package/src/plugins/eventual-consistency/consolidation.js +32 -36
- package/src/plugins/eventual-consistency/garbage-collection.js +11 -13
- package/src/plugins/eventual-consistency/index.js +28 -19
- package/src/plugins/eventual-consistency/install.js +9 -26
- package/src/plugins/eventual-consistency/partitions.js +5 -0
- package/src/plugins/eventual-consistency/transactions.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +36 -1
- package/src/plugins/fulltext.plugin.js +76 -22
- package/src/plugins/metrics.plugin.js +70 -20
- package/src/plugins/s3-queue.plugin.js +21 -120
- package/src/plugins/scheduler.plugin.js +11 -37
package/dist/s3db.es.js
CHANGED
|
@@ -833,18 +833,23 @@ class PluginStorage {
|
|
|
833
833
|
return `plugin=${this.pluginSlug}/${parts.join("/")}`;
|
|
834
834
|
}
|
|
835
835
|
/**
|
|
836
|
-
* Save data with metadata encoding and
|
|
836
|
+
* Save data with metadata encoding, behavior support, and optional TTL
|
|
837
837
|
*
|
|
838
838
|
* @param {string} key - S3 key
|
|
839
839
|
* @param {Object} data - Data to save
|
|
840
840
|
* @param {Object} options - Options
|
|
841
|
+
* @param {number} options.ttl - Time-to-live in seconds (optional)
|
|
841
842
|
* @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
|
|
842
843
|
* @param {string} options.contentType - Content type (default: application/json)
|
|
843
844
|
* @returns {Promise<void>}
|
|
844
845
|
*/
|
|
845
|
-
async
|
|
846
|
-
const { behavior = "body-overflow", contentType = "application/json" } = options;
|
|
847
|
-
const
|
|
846
|
+
async set(key, data, options = {}) {
|
|
847
|
+
const { ttl, behavior = "body-overflow", contentType = "application/json" } = options;
|
|
848
|
+
const dataToSave = { ...data };
|
|
849
|
+
if (ttl && typeof ttl === "number" && ttl > 0) {
|
|
850
|
+
dataToSave._expiresAt = Date.now() + ttl * 1e3;
|
|
851
|
+
}
|
|
852
|
+
const { metadata, body } = this._applyBehavior(dataToSave, behavior);
|
|
848
853
|
const putParams = {
|
|
849
854
|
key,
|
|
850
855
|
metadata,
|
|
@@ -855,14 +860,21 @@ class PluginStorage {
|
|
|
855
860
|
}
|
|
856
861
|
const [ok, err] = await tryFn(() => this.client.putObject(putParams));
|
|
857
862
|
if (!ok) {
|
|
858
|
-
throw new Error(`PluginStorage.
|
|
863
|
+
throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
|
|
859
864
|
}
|
|
860
865
|
}
|
|
861
866
|
/**
|
|
862
|
-
*
|
|
867
|
+
* Alias for set() to maintain backward compatibility
|
|
868
|
+
* @deprecated Use set() instead
|
|
869
|
+
*/
|
|
870
|
+
async put(key, data, options = {}) {
|
|
871
|
+
return this.set(key, data, options);
|
|
872
|
+
}
|
|
873
|
+
/**
|
|
874
|
+
* Get data with automatic metadata decoding and TTL check
|
|
863
875
|
*
|
|
864
876
|
* @param {string} key - S3 key
|
|
865
|
-
* @returns {Promise<Object|null>} Data or null if not found
|
|
877
|
+
* @returns {Promise<Object|null>} Data or null if not found/expired
|
|
866
878
|
*/
|
|
867
879
|
async get(key) {
|
|
868
880
|
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
@@ -874,18 +886,28 @@ class PluginStorage {
|
|
|
874
886
|
}
|
|
875
887
|
const metadata = response.Metadata || {};
|
|
876
888
|
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
889
|
+
let data = parsedMetadata;
|
|
877
890
|
if (response.Body) {
|
|
878
891
|
try {
|
|
879
892
|
const bodyContent = await response.Body.transformToString();
|
|
880
893
|
if (bodyContent && bodyContent.trim()) {
|
|
881
894
|
const body = JSON.parse(bodyContent);
|
|
882
|
-
|
|
895
|
+
data = { ...parsedMetadata, ...body };
|
|
883
896
|
}
|
|
884
897
|
} catch (parseErr) {
|
|
885
898
|
throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
|
|
886
899
|
}
|
|
887
900
|
}
|
|
888
|
-
|
|
901
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
902
|
+
if (expiresAt) {
|
|
903
|
+
if (Date.now() > expiresAt) {
|
|
904
|
+
await this.delete(key);
|
|
905
|
+
return null;
|
|
906
|
+
}
|
|
907
|
+
delete data._expiresat;
|
|
908
|
+
delete data._expiresAt;
|
|
909
|
+
}
|
|
910
|
+
return data;
|
|
889
911
|
}
|
|
890
912
|
/**
|
|
891
913
|
* Parse metadata values back to their original types
|
|
@@ -968,6 +990,123 @@ class PluginStorage {
|
|
|
968
990
|
if (!keyPrefix) return keys;
|
|
969
991
|
return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
|
|
970
992
|
}
|
|
993
|
+
/**
|
|
994
|
+
* Check if a key exists (not expired)
|
|
995
|
+
*
|
|
996
|
+
* @param {string} key - S3 key
|
|
997
|
+
* @returns {Promise<boolean>} True if exists and not expired
|
|
998
|
+
*/
|
|
999
|
+
async has(key) {
|
|
1000
|
+
const data = await this.get(key);
|
|
1001
|
+
return data !== null;
|
|
1002
|
+
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Check if a key is expired
|
|
1005
|
+
*
|
|
1006
|
+
* @param {string} key - S3 key
|
|
1007
|
+
* @returns {Promise<boolean>} True if expired or not found
|
|
1008
|
+
*/
|
|
1009
|
+
async isExpired(key) {
|
|
1010
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1011
|
+
if (!ok) {
|
|
1012
|
+
return true;
|
|
1013
|
+
}
|
|
1014
|
+
const metadata = response.Metadata || {};
|
|
1015
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1016
|
+
let data = parsedMetadata;
|
|
1017
|
+
if (response.Body) {
|
|
1018
|
+
try {
|
|
1019
|
+
const bodyContent = await response.Body.transformToString();
|
|
1020
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1021
|
+
const body = JSON.parse(bodyContent);
|
|
1022
|
+
data = { ...parsedMetadata, ...body };
|
|
1023
|
+
}
|
|
1024
|
+
} catch {
|
|
1025
|
+
return true;
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1029
|
+
if (!expiresAt) {
|
|
1030
|
+
return false;
|
|
1031
|
+
}
|
|
1032
|
+
return Date.now() > expiresAt;
|
|
1033
|
+
}
|
|
1034
|
+
/**
|
|
1035
|
+
* Get remaining TTL in seconds
|
|
1036
|
+
*
|
|
1037
|
+
* @param {string} key - S3 key
|
|
1038
|
+
* @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
|
|
1039
|
+
*/
|
|
1040
|
+
async getTTL(key) {
|
|
1041
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1042
|
+
if (!ok) {
|
|
1043
|
+
return null;
|
|
1044
|
+
}
|
|
1045
|
+
const metadata = response.Metadata || {};
|
|
1046
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1047
|
+
let data = parsedMetadata;
|
|
1048
|
+
if (response.Body) {
|
|
1049
|
+
try {
|
|
1050
|
+
const bodyContent = await response.Body.transformToString();
|
|
1051
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1052
|
+
const body = JSON.parse(bodyContent);
|
|
1053
|
+
data = { ...parsedMetadata, ...body };
|
|
1054
|
+
}
|
|
1055
|
+
} catch {
|
|
1056
|
+
return null;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1060
|
+
if (!expiresAt) {
|
|
1061
|
+
return null;
|
|
1062
|
+
}
|
|
1063
|
+
const remaining = Math.max(0, expiresAt - Date.now());
|
|
1064
|
+
return Math.floor(remaining / 1e3);
|
|
1065
|
+
}
|
|
1066
|
+
/**
|
|
1067
|
+
* Extend TTL by adding additional seconds
|
|
1068
|
+
*
|
|
1069
|
+
* @param {string} key - S3 key
|
|
1070
|
+
* @param {number} additionalSeconds - Seconds to add to current TTL
|
|
1071
|
+
* @returns {Promise<boolean>} True if extended, false if not found or no TTL
|
|
1072
|
+
*/
|
|
1073
|
+
async touch(key, additionalSeconds) {
|
|
1074
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1075
|
+
if (!ok) {
|
|
1076
|
+
return false;
|
|
1077
|
+
}
|
|
1078
|
+
const metadata = response.Metadata || {};
|
|
1079
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1080
|
+
let data = parsedMetadata;
|
|
1081
|
+
if (response.Body) {
|
|
1082
|
+
try {
|
|
1083
|
+
const bodyContent = await response.Body.transformToString();
|
|
1084
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1085
|
+
const body = JSON.parse(bodyContent);
|
|
1086
|
+
data = { ...parsedMetadata, ...body };
|
|
1087
|
+
}
|
|
1088
|
+
} catch {
|
|
1089
|
+
return false;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1093
|
+
if (!expiresAt) {
|
|
1094
|
+
return false;
|
|
1095
|
+
}
|
|
1096
|
+
data._expiresAt = expiresAt + additionalSeconds * 1e3;
|
|
1097
|
+
delete data._expiresat;
|
|
1098
|
+
const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, "body-overflow");
|
|
1099
|
+
const putParams = {
|
|
1100
|
+
key,
|
|
1101
|
+
metadata: newMetadata,
|
|
1102
|
+
contentType: "application/json"
|
|
1103
|
+
};
|
|
1104
|
+
if (newBody !== null) {
|
|
1105
|
+
putParams.body = JSON.stringify(newBody);
|
|
1106
|
+
}
|
|
1107
|
+
const [putOk] = await tryFn(() => this.client.putObject(putParams));
|
|
1108
|
+
return putOk;
|
|
1109
|
+
}
|
|
971
1110
|
/**
|
|
972
1111
|
* Delete a single object
|
|
973
1112
|
*
|
|
@@ -1045,6 +1184,78 @@ class PluginStorage {
|
|
|
1045
1184
|
}
|
|
1046
1185
|
return results;
|
|
1047
1186
|
}
|
|
1187
|
+
/**
|
|
1188
|
+
* Acquire a distributed lock with TTL and retry logic
|
|
1189
|
+
*
|
|
1190
|
+
* @param {string} lockName - Lock identifier
|
|
1191
|
+
* @param {Object} options - Lock options
|
|
1192
|
+
* @param {number} options.ttl - Lock TTL in seconds (default: 30)
|
|
1193
|
+
* @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
|
|
1194
|
+
* @param {string} options.workerId - Worker identifier (default: 'unknown')
|
|
1195
|
+
* @returns {Promise<Object|null>} Lock object or null if couldn't acquire
|
|
1196
|
+
*/
|
|
1197
|
+
async acquireLock(lockName, options = {}) {
|
|
1198
|
+
const { ttl = 30, timeout = 0, workerId = "unknown" } = options;
|
|
1199
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1200
|
+
const startTime = Date.now();
|
|
1201
|
+
while (true) {
|
|
1202
|
+
const existing = await this.get(key);
|
|
1203
|
+
if (!existing) {
|
|
1204
|
+
await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
|
|
1205
|
+
return { key, workerId };
|
|
1206
|
+
}
|
|
1207
|
+
if (Date.now() - startTime >= timeout) {
|
|
1208
|
+
return null;
|
|
1209
|
+
}
|
|
1210
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
/**
|
|
1214
|
+
* Release a distributed lock
|
|
1215
|
+
*
|
|
1216
|
+
* @param {string} lockName - Lock identifier
|
|
1217
|
+
* @returns {Promise<void>}
|
|
1218
|
+
*/
|
|
1219
|
+
async releaseLock(lockName) {
|
|
1220
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1221
|
+
await this.delete(key);
|
|
1222
|
+
}
|
|
1223
|
+
/**
|
|
1224
|
+
* Check if a lock is currently held
|
|
1225
|
+
*
|
|
1226
|
+
* @param {string} lockName - Lock identifier
|
|
1227
|
+
* @returns {Promise<boolean>} True if locked
|
|
1228
|
+
*/
|
|
1229
|
+
async isLocked(lockName) {
|
|
1230
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1231
|
+
const lock = await this.get(key);
|
|
1232
|
+
return lock !== null;
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Increment a counter value
|
|
1236
|
+
*
|
|
1237
|
+
* @param {string} key - S3 key
|
|
1238
|
+
* @param {number} amount - Amount to increment (default: 1)
|
|
1239
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1240
|
+
* @returns {Promise<number>} New value
|
|
1241
|
+
*/
|
|
1242
|
+
async increment(key, amount = 1, options = {}) {
|
|
1243
|
+
const data = await this.get(key);
|
|
1244
|
+
const value = (data?.value || 0) + amount;
|
|
1245
|
+
await this.set(key, { value }, options);
|
|
1246
|
+
return value;
|
|
1247
|
+
}
|
|
1248
|
+
/**
|
|
1249
|
+
* Decrement a counter value
|
|
1250
|
+
*
|
|
1251
|
+
* @param {string} key - S3 key
|
|
1252
|
+
* @param {number} amount - Amount to decrement (default: 1)
|
|
1253
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1254
|
+
* @returns {Promise<number>} New value
|
|
1255
|
+
*/
|
|
1256
|
+
async decrement(key, amount = 1, options = {}) {
|
|
1257
|
+
return this.increment(key, -amount, options);
|
|
1258
|
+
}
|
|
1048
1259
|
/**
|
|
1049
1260
|
* Apply behavior to split data between metadata and body
|
|
1050
1261
|
*
|
|
@@ -1348,12 +1559,18 @@ class AuditPlugin extends Plugin {
|
|
|
1348
1559
|
recordId: "string|required",
|
|
1349
1560
|
userId: "string|optional",
|
|
1350
1561
|
timestamp: "string|required",
|
|
1562
|
+
createdAt: "string|required",
|
|
1563
|
+
// YYYY-MM-DD for partitioning
|
|
1351
1564
|
oldData: "string|optional",
|
|
1352
1565
|
newData: "string|optional",
|
|
1353
1566
|
partition: "string|optional",
|
|
1354
1567
|
partitionValues: "string|optional",
|
|
1355
1568
|
metadata: "string|optional"
|
|
1356
1569
|
},
|
|
1570
|
+
partitions: {
|
|
1571
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } },
|
|
1572
|
+
byResource: { fields: { resourceName: "string" } }
|
|
1573
|
+
},
|
|
1357
1574
|
behavior: "body-overflow"
|
|
1358
1575
|
}));
|
|
1359
1576
|
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
@@ -1457,10 +1674,13 @@ class AuditPlugin extends Plugin {
|
|
|
1457
1674
|
if (!this.auditResource) {
|
|
1458
1675
|
return;
|
|
1459
1676
|
}
|
|
1677
|
+
const now = /* @__PURE__ */ new Date();
|
|
1460
1678
|
const auditRecord = {
|
|
1461
1679
|
id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
|
|
1462
1680
|
userId: this.getCurrentUserId?.() || "system",
|
|
1463
|
-
timestamp:
|
|
1681
|
+
timestamp: now.toISOString(),
|
|
1682
|
+
createdAt: now.toISOString().slice(0, 10),
|
|
1683
|
+
// YYYY-MM-DD for partitioning
|
|
1464
1684
|
metadata: JSON.stringify({ source: "audit-plugin", version: "2.0" }),
|
|
1465
1685
|
resourceName: auditData.resourceName,
|
|
1466
1686
|
operation: auditData.operation,
|
|
@@ -1535,9 +1755,25 @@ class AuditPlugin extends Plugin {
|
|
|
1535
1755
|
async getAuditLogs(options = {}) {
|
|
1536
1756
|
if (!this.auditResource) return [];
|
|
1537
1757
|
const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
|
|
1538
|
-
const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
|
|
1539
1758
|
let items = [];
|
|
1540
|
-
if (
|
|
1759
|
+
if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
|
|
1760
|
+
const [ok, err, result] = await tryFn(
|
|
1761
|
+
() => this.auditResource.query({ resourceName }, { limit: limit + offset })
|
|
1762
|
+
);
|
|
1763
|
+
items = ok && result ? result : [];
|
|
1764
|
+
return items.slice(offset, offset + limit);
|
|
1765
|
+
} else if (startDate && !resourceName && !operation && !recordId && !partition) {
|
|
1766
|
+
const dates = this._generateDateRange(startDate, endDate);
|
|
1767
|
+
for (const date of dates) {
|
|
1768
|
+
const [ok, err, result] = await tryFn(
|
|
1769
|
+
() => this.auditResource.query({ createdAt: date })
|
|
1770
|
+
);
|
|
1771
|
+
if (ok && result) {
|
|
1772
|
+
items.push(...result);
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
return items.slice(offset, offset + limit);
|
|
1776
|
+
} else if (resourceName || operation || recordId || partition || startDate || endDate) {
|
|
1541
1777
|
const fetchSize = Math.min(1e4, Math.max(1e3, (limit + offset) * 20));
|
|
1542
1778
|
const result = await this.auditResource.list({ limit: fetchSize });
|
|
1543
1779
|
items = result || [];
|
|
@@ -1567,6 +1803,15 @@ class AuditPlugin extends Plugin {
|
|
|
1567
1803
|
return result.items || [];
|
|
1568
1804
|
}
|
|
1569
1805
|
}
|
|
1806
|
+
_generateDateRange(startDate, endDate) {
|
|
1807
|
+
const dates = [];
|
|
1808
|
+
const start = new Date(startDate);
|
|
1809
|
+
const end = endDate ? new Date(endDate) : /* @__PURE__ */ new Date();
|
|
1810
|
+
for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
|
|
1811
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
1812
|
+
}
|
|
1813
|
+
return dates;
|
|
1814
|
+
}
|
|
1570
1815
|
async getRecordHistory(resourceName, recordId) {
|
|
1571
1816
|
return await this.getAuditLogs({ resourceName, recordId });
|
|
1572
1817
|
}
|
|
@@ -1599,6 +1844,37 @@ class AuditPlugin extends Plugin {
|
|
|
1599
1844
|
}
|
|
1600
1845
|
return stats;
|
|
1601
1846
|
}
|
|
1847
|
+
/**
|
|
1848
|
+
* Clean up audit logs older than retention period
|
|
1849
|
+
* @param {number} retentionDays - Number of days to retain (default: 90)
|
|
1850
|
+
* @returns {Promise<number>} Number of records deleted
|
|
1851
|
+
*/
|
|
1852
|
+
async cleanupOldAudits(retentionDays = 90) {
|
|
1853
|
+
if (!this.auditResource) return 0;
|
|
1854
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
1855
|
+
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
|
1856
|
+
const datesToDelete = [];
|
|
1857
|
+
const startDate = new Date(cutoffDate);
|
|
1858
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
1859
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
1860
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
1861
|
+
}
|
|
1862
|
+
let deletedCount = 0;
|
|
1863
|
+
for (const dateStr of datesToDelete) {
|
|
1864
|
+
const [ok, err, oldAudits] = await tryFn(
|
|
1865
|
+
() => this.auditResource.query({ createdAt: dateStr })
|
|
1866
|
+
);
|
|
1867
|
+
if (ok && oldAudits) {
|
|
1868
|
+
for (const audit of oldAudits) {
|
|
1869
|
+
const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
|
|
1870
|
+
if (delOk) {
|
|
1871
|
+
deletedCount++;
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
return deletedCount;
|
|
1877
|
+
}
|
|
1602
1878
|
}
|
|
1603
1879
|
|
|
1604
1880
|
class BaseBackupDriver {
|
|
@@ -4834,6 +5110,8 @@ function createConfig(options, detectedTimezone) {
|
|
|
4834
5110
|
consolidationWindow: consolidation.window ?? 24,
|
|
4835
5111
|
autoConsolidate: consolidation.auto !== false,
|
|
4836
5112
|
mode: consolidation.mode || "async",
|
|
5113
|
+
// ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
|
|
5114
|
+
markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
|
|
4837
5115
|
// Late arrivals
|
|
4838
5116
|
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
4839
5117
|
// Batch transactions
|
|
@@ -4937,6 +5215,21 @@ function getTimezoneOffset(timezone, verbose = false) {
|
|
|
4937
5215
|
return offsets[timezone] || 0;
|
|
4938
5216
|
}
|
|
4939
5217
|
}
|
|
5218
|
+
function getISOWeek(date) {
|
|
5219
|
+
const target = new Date(date.valueOf());
|
|
5220
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
5221
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
5222
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
5223
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
5224
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
5225
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
5226
|
+
}
|
|
5227
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
5228
|
+
return {
|
|
5229
|
+
year: target.getUTCFullYear(),
|
|
5230
|
+
week: weekNumber
|
|
5231
|
+
};
|
|
5232
|
+
}
|
|
4940
5233
|
function getCohortInfo(date, timezone, verbose = false) {
|
|
4941
5234
|
const offset = getTimezoneOffset(timezone, verbose);
|
|
4942
5235
|
const localDate = new Date(date.getTime() + offset);
|
|
@@ -4944,10 +5237,14 @@ function getCohortInfo(date, timezone, verbose = false) {
|
|
|
4944
5237
|
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4945
5238
|
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4946
5239
|
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
5240
|
+
const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
|
|
5241
|
+
const week = `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
4947
5242
|
return {
|
|
4948
5243
|
date: `${year}-${month}-${day}`,
|
|
4949
5244
|
hour: `${year}-${month}-${day}T${hour}`,
|
|
4950
5245
|
// ISO-like format for hour partition
|
|
5246
|
+
week,
|
|
5247
|
+
// ISO 8601 week format (e.g., '2025-W42')
|
|
4951
5248
|
month: `${year}-${month}`
|
|
4952
5249
|
};
|
|
4953
5250
|
}
|
|
@@ -5025,6 +5322,11 @@ function createPartitionConfig() {
|
|
|
5025
5322
|
cohortDate: "string"
|
|
5026
5323
|
}
|
|
5027
5324
|
},
|
|
5325
|
+
byWeek: {
|
|
5326
|
+
fields: {
|
|
5327
|
+
cohortWeek: "string"
|
|
5328
|
+
}
|
|
5329
|
+
},
|
|
5028
5330
|
byMonth: {
|
|
5029
5331
|
fields: {
|
|
5030
5332
|
cohortMonth: "string"
|
|
@@ -5064,6 +5366,7 @@ async function createTransaction(handler, data, config) {
|
|
|
5064
5366
|
timestamp: now.toISOString(),
|
|
5065
5367
|
cohortDate: cohortInfo.date,
|
|
5066
5368
|
cohortHour: cohortInfo.hour,
|
|
5369
|
+
cohortWeek: cohortInfo.week,
|
|
5067
5370
|
cohortMonth: cohortInfo.month,
|
|
5068
5371
|
source: data.source || "unknown",
|
|
5069
5372
|
applied: false
|
|
@@ -5104,50 +5407,6 @@ async function flushPendingTransactions(handler) {
|
|
|
5104
5407
|
}
|
|
5105
5408
|
}
|
|
5106
5409
|
|
|
5107
|
-
async function cleanupStaleLocks(lockResource, config) {
|
|
5108
|
-
const now = Date.now();
|
|
5109
|
-
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
5110
|
-
const cutoffTime = now - lockTimeoutMs;
|
|
5111
|
-
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
5112
|
-
const [lockAcquired] = await tryFn(
|
|
5113
|
-
() => lockResource.insert({
|
|
5114
|
-
id: cleanupLockId,
|
|
5115
|
-
lockedAt: Date.now(),
|
|
5116
|
-
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5117
|
-
})
|
|
5118
|
-
);
|
|
5119
|
-
if (!lockAcquired) {
|
|
5120
|
-
if (config.verbose) {
|
|
5121
|
-
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
5122
|
-
}
|
|
5123
|
-
return;
|
|
5124
|
-
}
|
|
5125
|
-
try {
|
|
5126
|
-
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
5127
|
-
if (!ok || !locks || locks.length === 0) return;
|
|
5128
|
-
const staleLocks = locks.filter(
|
|
5129
|
-
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
5130
|
-
);
|
|
5131
|
-
if (staleLocks.length === 0) return;
|
|
5132
|
-
if (config.verbose) {
|
|
5133
|
-
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
5134
|
-
}
|
|
5135
|
-
const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
5136
|
-
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
5137
|
-
return deleted;
|
|
5138
|
-
});
|
|
5139
|
-
if (errors && errors.length > 0 && config.verbose) {
|
|
5140
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5141
|
-
}
|
|
5142
|
-
} catch (error) {
|
|
5143
|
-
if (config.verbose) {
|
|
5144
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5145
|
-
}
|
|
5146
|
-
} finally {
|
|
5147
|
-
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
5148
|
-
}
|
|
5149
|
-
}
|
|
5150
|
-
|
|
5151
5410
|
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
5152
5411
|
const intervalMs = config.consolidationInterval * 1e3;
|
|
5153
5412
|
if (config.verbose) {
|
|
@@ -5244,17 +5503,15 @@ async function runConsolidation(transactionResource, consolidateRecordFn, emitFn
|
|
|
5244
5503
|
}
|
|
5245
5504
|
}
|
|
5246
5505
|
}
|
|
5247
|
-
async function consolidateRecord(originalId, transactionResource, targetResource,
|
|
5248
|
-
|
|
5249
|
-
const
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
);
|
|
5257
|
-
if (!lockAcquired) {
|
|
5506
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, storage, analyticsResource, updateAnalyticsFn, config) {
|
|
5507
|
+
const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
|
|
5508
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5509
|
+
ttl: config.lockTimeout || 30,
|
|
5510
|
+
timeout: 0,
|
|
5511
|
+
// Don't wait if locked
|
|
5512
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5513
|
+
});
|
|
5514
|
+
if (!lock) {
|
|
5258
5515
|
if (config.verbose) {
|
|
5259
5516
|
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
5260
5517
|
}
|
|
@@ -5465,7 +5722,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5465
5722
|
}
|
|
5466
5723
|
if (updateOk) {
|
|
5467
5724
|
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5468
|
-
const
|
|
5725
|
+
const markAppliedConcurrency = config.markAppliedConcurrency || 50;
|
|
5726
|
+
const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
|
|
5469
5727
|
const [ok2, err2] = await tryFn(
|
|
5470
5728
|
() => transactionResource.update(txn.id, { applied: true })
|
|
5471
5729
|
);
|
|
@@ -5513,9 +5771,11 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5513
5771
|
}
|
|
5514
5772
|
return consolidatedValue;
|
|
5515
5773
|
} finally {
|
|
5516
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5774
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5775
|
+
() => storage.releaseLock(lockKey)
|
|
5776
|
+
);
|
|
5517
5777
|
if (!lockReleased && config.verbose) {
|
|
5518
|
-
console.warn(`[EventualConsistency] Failed to release lock ${
|
|
5778
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5519
5779
|
}
|
|
5520
5780
|
}
|
|
5521
5781
|
}
|
|
@@ -5589,17 +5849,15 @@ async function getCohortStats(cohortDate, transactionResource) {
|
|
|
5589
5849
|
}
|
|
5590
5850
|
return stats;
|
|
5591
5851
|
}
|
|
5592
|
-
async function recalculateRecord(originalId, transactionResource, targetResource,
|
|
5593
|
-
|
|
5594
|
-
const
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
);
|
|
5602
|
-
if (!lockAcquired) {
|
|
5852
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, storage, consolidateRecordFn, config) {
|
|
5853
|
+
const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
|
|
5854
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5855
|
+
ttl: config.lockTimeout || 30,
|
|
5856
|
+
timeout: 0,
|
|
5857
|
+
// Don't wait if locked
|
|
5858
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5859
|
+
});
|
|
5860
|
+
if (!lock) {
|
|
5603
5861
|
if (config.verbose) {
|
|
5604
5862
|
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5605
5863
|
}
|
|
@@ -5667,9 +5925,11 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5667
5925
|
}
|
|
5668
5926
|
return consolidatedValue;
|
|
5669
5927
|
} finally {
|
|
5670
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5928
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5929
|
+
() => storage.releaseLock(lockKey)
|
|
5930
|
+
);
|
|
5671
5931
|
if (!lockReleased && config.verbose) {
|
|
5672
|
-
console.warn(`[EventualConsistency] Failed to release recalculate lock ${
|
|
5932
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5673
5933
|
}
|
|
5674
5934
|
}
|
|
5675
5935
|
}
|
|
@@ -5681,16 +5941,16 @@ function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCall
|
|
|
5681
5941
|
}, gcIntervalMs);
|
|
5682
5942
|
return handler.gcTimer;
|
|
5683
5943
|
}
|
|
5684
|
-
async function runGarbageCollection(transactionResource,
|
|
5685
|
-
const
|
|
5686
|
-
const
|
|
5687
|
-
|
|
5688
|
-
|
|
5689
|
-
|
|
5690
|
-
|
|
5691
|
-
|
|
5692
|
-
);
|
|
5693
|
-
if (!
|
|
5944
|
+
async function runGarbageCollection(transactionResource, storage, config, emitFn) {
|
|
5945
|
+
const lockKey = `gc-${config.resource}-${config.field}`;
|
|
5946
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5947
|
+
ttl: 300,
|
|
5948
|
+
// 5 minutes for GC
|
|
5949
|
+
timeout: 0,
|
|
5950
|
+
// Don't wait if locked
|
|
5951
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5952
|
+
});
|
|
5953
|
+
if (!lock) {
|
|
5694
5954
|
if (config.verbose) {
|
|
5695
5955
|
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5696
5956
|
}
|
|
@@ -5748,7 +6008,7 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
|
|
|
5748
6008
|
emitFn("eventual-consistency.gc-error", error);
|
|
5749
6009
|
}
|
|
5750
6010
|
} finally {
|
|
5751
|
-
await tryFn(() =>
|
|
6011
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
5752
6012
|
}
|
|
5753
6013
|
}
|
|
5754
6014
|
|
|
@@ -5773,22 +6033,26 @@ AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
|
5773
6033
|
const cohortCount = Object.keys(byHour).length;
|
|
5774
6034
|
if (config.verbose) {
|
|
5775
6035
|
console.log(
|
|
5776
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
6036
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
|
|
5777
6037
|
);
|
|
5778
6038
|
}
|
|
5779
|
-
|
|
5780
|
-
|
|
5781
|
-
|
|
6039
|
+
await Promise.all(
|
|
6040
|
+
Object.entries(byHour).map(
|
|
6041
|
+
([cohort, txns]) => upsertAnalytics("hour", cohort, txns, analyticsResource, config)
|
|
6042
|
+
)
|
|
6043
|
+
);
|
|
5782
6044
|
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5783
6045
|
const uniqueHours = Object.keys(byHour);
|
|
5784
6046
|
if (config.verbose) {
|
|
5785
6047
|
console.log(
|
|
5786
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
6048
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
|
|
5787
6049
|
);
|
|
5788
6050
|
}
|
|
5789
|
-
|
|
5790
|
-
|
|
5791
|
-
|
|
6051
|
+
await Promise.all(
|
|
6052
|
+
uniqueHours.map(
|
|
6053
|
+
(cohortHour) => rollupAnalytics(cohortHour, analyticsResource, config)
|
|
6054
|
+
)
|
|
6055
|
+
);
|
|
5792
6056
|
}
|
|
5793
6057
|
if (config.verbose) {
|
|
5794
6058
|
console.log(
|
|
@@ -5891,18 +6155,53 @@ function calculateOperationBreakdown(transactions) {
|
|
|
5891
6155
|
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5892
6156
|
const cohortDate = cohortHour.substring(0, 10);
|
|
5893
6157
|
const cohortMonth = cohortHour.substring(0, 7);
|
|
6158
|
+
const date = new Date(cohortDate);
|
|
6159
|
+
const cohortWeek = getCohortWeekFromDate(date);
|
|
5894
6160
|
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
|
|
6161
|
+
await rollupPeriod("week", cohortWeek, cohortWeek, analyticsResource, config);
|
|
5895
6162
|
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
|
|
5896
6163
|
}
|
|
6164
|
+
function getCohortWeekFromDate(date) {
|
|
6165
|
+
const target = new Date(date.valueOf());
|
|
6166
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
6167
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
6168
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
6169
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
6170
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
6171
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
6172
|
+
}
|
|
6173
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
6174
|
+
const weekYear = target.getUTCFullYear();
|
|
6175
|
+
return `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
6176
|
+
}
|
|
5897
6177
|
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5898
|
-
|
|
6178
|
+
let sourcePeriod;
|
|
6179
|
+
if (period === "day") {
|
|
6180
|
+
sourcePeriod = "hour";
|
|
6181
|
+
} else if (period === "week") {
|
|
6182
|
+
sourcePeriod = "day";
|
|
6183
|
+
} else if (period === "month") {
|
|
6184
|
+
sourcePeriod = "week";
|
|
6185
|
+
} else {
|
|
6186
|
+
sourcePeriod = "day";
|
|
6187
|
+
}
|
|
5899
6188
|
const [ok, err, allAnalytics] = await tryFn(
|
|
5900
6189
|
() => analyticsResource.list()
|
|
5901
6190
|
);
|
|
5902
6191
|
if (!ok || !allAnalytics) return;
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
6192
|
+
let sourceAnalytics;
|
|
6193
|
+
if (period === "week") {
|
|
6194
|
+
sourceAnalytics = allAnalytics.filter((a) => {
|
|
6195
|
+
if (a.period !== sourcePeriod) return false;
|
|
6196
|
+
const dayDate = new Date(a.cohort);
|
|
6197
|
+
const dayWeek = getCohortWeekFromDate(dayDate);
|
|
6198
|
+
return dayWeek === cohort;
|
|
6199
|
+
});
|
|
6200
|
+
} else {
|
|
6201
|
+
sourceAnalytics = allAnalytics.filter(
|
|
6202
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
6203
|
+
);
|
|
6204
|
+
}
|
|
5906
6205
|
if (sourceAnalytics.length === 0) return;
|
|
5907
6206
|
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5908
6207
|
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
@@ -6111,6 +6410,32 @@ async function getYearByMonth(resourceName, field, year, options, fieldHandlers)
|
|
|
6111
6410
|
}
|
|
6112
6411
|
return data;
|
|
6113
6412
|
}
|
|
6413
|
+
async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
|
|
6414
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6415
|
+
period: "week",
|
|
6416
|
+
year
|
|
6417
|
+
}, fieldHandlers);
|
|
6418
|
+
if (options.fillGaps) {
|
|
6419
|
+
const startWeek = `${year}-W01`;
|
|
6420
|
+
const endWeek = `${year}-W53`;
|
|
6421
|
+
return fillGaps(data, "week", startWeek, endWeek);
|
|
6422
|
+
}
|
|
6423
|
+
return data;
|
|
6424
|
+
}
|
|
6425
|
+
async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
|
|
6426
|
+
const year = parseInt(month.substring(0, 4));
|
|
6427
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
6428
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
6429
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
6430
|
+
const firstWeek = getCohortWeekFromDate(firstDay);
|
|
6431
|
+
const lastWeek = getCohortWeekFromDate(lastDay);
|
|
6432
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6433
|
+
period: "week",
|
|
6434
|
+
startDate: firstWeek,
|
|
6435
|
+
endDate: lastWeek
|
|
6436
|
+
}, fieldHandlers);
|
|
6437
|
+
return data;
|
|
6438
|
+
}
|
|
6114
6439
|
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
6115
6440
|
let year, monthNum;
|
|
6116
6441
|
if (month === "last") {
|
|
@@ -6338,7 +6663,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6338
6663
|
if (!handler.targetResource) return;
|
|
6339
6664
|
const resourceName = handler.resource;
|
|
6340
6665
|
const fieldName = handler.field;
|
|
6341
|
-
const transactionResourceName =
|
|
6666
|
+
const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
|
|
6342
6667
|
const partitionConfig = createPartitionConfig();
|
|
6343
6668
|
const [ok, err, transactionResource] = await tryFn(
|
|
6344
6669
|
() => database.createResource({
|
|
@@ -6352,6 +6677,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6352
6677
|
timestamp: "string|required",
|
|
6353
6678
|
cohortDate: "string|required",
|
|
6354
6679
|
cohortHour: "string|required",
|
|
6680
|
+
cohortWeek: "string|optional",
|
|
6355
6681
|
cohortMonth: "string|optional",
|
|
6356
6682
|
source: "string|optional",
|
|
6357
6683
|
applied: "boolean|optional"
|
|
@@ -6367,36 +6693,18 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6367
6693
|
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
6368
6694
|
}
|
|
6369
6695
|
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
6370
|
-
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
6371
|
-
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
6372
|
-
() => database.createResource({
|
|
6373
|
-
name: lockResourceName,
|
|
6374
|
-
attributes: {
|
|
6375
|
-
id: "string|required",
|
|
6376
|
-
lockedAt: "number|required",
|
|
6377
|
-
workerId: "string|optional"
|
|
6378
|
-
},
|
|
6379
|
-
behavior: "body-only",
|
|
6380
|
-
timestamps: false,
|
|
6381
|
-
createdBy: "EventualConsistencyPlugin"
|
|
6382
|
-
})
|
|
6383
|
-
);
|
|
6384
|
-
if (!lockOk && !database.resources[lockResourceName]) {
|
|
6385
|
-
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
6386
|
-
}
|
|
6387
|
-
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
6388
6696
|
if (config.enableAnalytics) {
|
|
6389
6697
|
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
6390
6698
|
}
|
|
6391
6699
|
addHelperMethodsForHandler(handler, plugin, config);
|
|
6392
6700
|
if (config.verbose) {
|
|
6393
6701
|
console.log(
|
|
6394
|
-
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}
|
|
6702
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ""} (locks via PluginStorage TTL)`
|
|
6395
6703
|
);
|
|
6396
6704
|
}
|
|
6397
6705
|
}
|
|
6398
6706
|
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
6399
|
-
const analyticsResourceName =
|
|
6707
|
+
const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
|
|
6400
6708
|
const [ok, err, analyticsResource] = await tryFn(
|
|
6401
6709
|
() => database.createResource({
|
|
6402
6710
|
name: analyticsResourceName,
|
|
@@ -6570,7 +6878,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6570
6878
|
originalId,
|
|
6571
6879
|
this.transactionResource,
|
|
6572
6880
|
this.targetResource,
|
|
6573
|
-
this.
|
|
6881
|
+
this.getStorage(),
|
|
6574
6882
|
this.analyticsResource,
|
|
6575
6883
|
(transactions) => this.updateAnalytics(transactions),
|
|
6576
6884
|
this.config
|
|
@@ -6606,7 +6914,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6606
6914
|
originalId,
|
|
6607
6915
|
this.transactionResource,
|
|
6608
6916
|
this.targetResource,
|
|
6609
|
-
this.
|
|
6917
|
+
this.getStorage(),
|
|
6610
6918
|
(id) => this.consolidateRecord(id),
|
|
6611
6919
|
this.config
|
|
6612
6920
|
);
|
|
@@ -6627,20 +6935,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6627
6935
|
const oldField = this.config.field;
|
|
6628
6936
|
const oldTransactionResource = this.transactionResource;
|
|
6629
6937
|
const oldTargetResource = this.targetResource;
|
|
6630
|
-
const oldLockResource = this.lockResource;
|
|
6631
6938
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6632
6939
|
this.config.resource = handler.resource;
|
|
6633
6940
|
this.config.field = handler.field;
|
|
6634
6941
|
this.transactionResource = handler.transactionResource;
|
|
6635
6942
|
this.targetResource = handler.targetResource;
|
|
6636
|
-
this.lockResource = handler.lockResource;
|
|
6637
6943
|
this.analyticsResource = handler.analyticsResource;
|
|
6638
6944
|
const result = await this.consolidateRecord(id);
|
|
6639
6945
|
this.config.resource = oldResource;
|
|
6640
6946
|
this.config.field = oldField;
|
|
6641
6947
|
this.transactionResource = oldTransactionResource;
|
|
6642
6948
|
this.targetResource = oldTargetResource;
|
|
6643
|
-
this.lockResource = oldLockResource;
|
|
6644
6949
|
this.analyticsResource = oldAnalyticsResource;
|
|
6645
6950
|
return result;
|
|
6646
6951
|
}
|
|
@@ -6653,20 +6958,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6653
6958
|
const oldField = this.config.field;
|
|
6654
6959
|
const oldTransactionResource = this.transactionResource;
|
|
6655
6960
|
const oldTargetResource = this.targetResource;
|
|
6656
|
-
const oldLockResource = this.lockResource;
|
|
6657
6961
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6658
6962
|
this.config.resource = handler.resource;
|
|
6659
6963
|
this.config.field = handler.field;
|
|
6660
6964
|
this.transactionResource = handler.transactionResource;
|
|
6661
6965
|
this.targetResource = handler.targetResource;
|
|
6662
|
-
this.lockResource = handler.lockResource;
|
|
6663
6966
|
this.analyticsResource = handler.analyticsResource;
|
|
6664
6967
|
const result = await this.consolidateRecord(id);
|
|
6665
6968
|
this.config.resource = oldResource;
|
|
6666
6969
|
this.config.field = oldField;
|
|
6667
6970
|
this.transactionResource = oldTransactionResource;
|
|
6668
6971
|
this.targetResource = oldTargetResource;
|
|
6669
|
-
this.lockResource = oldLockResource;
|
|
6670
6972
|
this.analyticsResource = oldAnalyticsResource;
|
|
6671
6973
|
return result;
|
|
6672
6974
|
}
|
|
@@ -6699,20 +7001,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6699
7001
|
const oldField = this.config.field;
|
|
6700
7002
|
const oldTransactionResource = this.transactionResource;
|
|
6701
7003
|
const oldTargetResource = this.targetResource;
|
|
6702
|
-
const oldLockResource = this.lockResource;
|
|
6703
7004
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6704
7005
|
this.config.resource = handler.resource;
|
|
6705
7006
|
this.config.field = handler.field;
|
|
6706
7007
|
this.transactionResource = handler.transactionResource;
|
|
6707
7008
|
this.targetResource = handler.targetResource;
|
|
6708
|
-
this.lockResource = handler.lockResource;
|
|
6709
7009
|
this.analyticsResource = handler.analyticsResource;
|
|
6710
7010
|
const result = await this.recalculateRecord(id);
|
|
6711
7011
|
this.config.resource = oldResource;
|
|
6712
7012
|
this.config.field = oldField;
|
|
6713
7013
|
this.transactionResource = oldTransactionResource;
|
|
6714
7014
|
this.targetResource = oldTargetResource;
|
|
6715
|
-
this.lockResource = oldLockResource;
|
|
6716
7015
|
this.analyticsResource = oldAnalyticsResource;
|
|
6717
7016
|
return result;
|
|
6718
7017
|
}
|
|
@@ -6725,13 +7024,11 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6725
7024
|
const oldField = this.config.field;
|
|
6726
7025
|
const oldTransactionResource = this.transactionResource;
|
|
6727
7026
|
const oldTargetResource = this.targetResource;
|
|
6728
|
-
const oldLockResource = this.lockResource;
|
|
6729
7027
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6730
7028
|
this.config.resource = resourceName;
|
|
6731
7029
|
this.config.field = fieldName;
|
|
6732
7030
|
this.transactionResource = handler.transactionResource;
|
|
6733
7031
|
this.targetResource = handler.targetResource;
|
|
6734
|
-
this.lockResource = handler.lockResource;
|
|
6735
7032
|
this.analyticsResource = handler.analyticsResource;
|
|
6736
7033
|
try {
|
|
6737
7034
|
await runConsolidation(
|
|
@@ -6745,7 +7042,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6745
7042
|
this.config.field = oldField;
|
|
6746
7043
|
this.transactionResource = oldTransactionResource;
|
|
6747
7044
|
this.targetResource = oldTargetResource;
|
|
6748
|
-
this.lockResource = oldLockResource;
|
|
6749
7045
|
this.analyticsResource = oldAnalyticsResource;
|
|
6750
7046
|
}
|
|
6751
7047
|
}
|
|
@@ -6758,16 +7054,14 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6758
7054
|
const oldField = this.config.field;
|
|
6759
7055
|
const oldTransactionResource = this.transactionResource;
|
|
6760
7056
|
const oldTargetResource = this.targetResource;
|
|
6761
|
-
const oldLockResource = this.lockResource;
|
|
6762
7057
|
this.config.resource = resourceName;
|
|
6763
7058
|
this.config.field = fieldName;
|
|
6764
7059
|
this.transactionResource = handler.transactionResource;
|
|
6765
7060
|
this.targetResource = handler.targetResource;
|
|
6766
|
-
this.lockResource = handler.lockResource;
|
|
6767
7061
|
try {
|
|
6768
7062
|
await runGarbageCollection(
|
|
6769
7063
|
this.transactionResource,
|
|
6770
|
-
this.
|
|
7064
|
+
this.getStorage(),
|
|
6771
7065
|
this.config,
|
|
6772
7066
|
(event, data) => this.emit(event, data)
|
|
6773
7067
|
);
|
|
@@ -6776,7 +7070,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6776
7070
|
this.config.field = oldField;
|
|
6777
7071
|
this.transactionResource = oldTransactionResource;
|
|
6778
7072
|
this.targetResource = oldTargetResource;
|
|
6779
|
-
this.lockResource = oldLockResource;
|
|
6780
7073
|
}
|
|
6781
7074
|
}
|
|
6782
7075
|
// Public Analytics API
|
|
@@ -6845,6 +7138,28 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6845
7138
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
6846
7139
|
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6847
7140
|
}
|
|
7141
|
+
/**
|
|
7142
|
+
* Get analytics for entire year, broken down by weeks
|
|
7143
|
+
* @param {string} resourceName - Resource name
|
|
7144
|
+
* @param {string} field - Field name
|
|
7145
|
+
* @param {number} year - Year (e.g., 2025)
|
|
7146
|
+
* @param {Object} options - Options
|
|
7147
|
+
* @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
|
|
7148
|
+
*/
|
|
7149
|
+
async getYearByWeek(resourceName, field, year, options = {}) {
|
|
7150
|
+
return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
|
|
7151
|
+
}
|
|
7152
|
+
/**
|
|
7153
|
+
* Get analytics for entire month, broken down by weeks
|
|
7154
|
+
* @param {string} resourceName - Resource name
|
|
7155
|
+
* @param {string} field - Field name
|
|
7156
|
+
* @param {string} month - Month in YYYY-MM format
|
|
7157
|
+
* @param {Object} options - Options
|
|
7158
|
+
* @returns {Promise<Array>} Weekly analytics for the month
|
|
7159
|
+
*/
|
|
7160
|
+
async getMonthByWeek(resourceName, field, month, options = {}) {
|
|
7161
|
+
return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
|
|
7162
|
+
}
|
|
6848
7163
|
/**
|
|
6849
7164
|
* Get top records by volume
|
|
6850
7165
|
* @param {string} resourceName - Resource name
|
|
@@ -6867,6 +7182,8 @@ class FullTextPlugin extends Plugin {
|
|
|
6867
7182
|
...options
|
|
6868
7183
|
};
|
|
6869
7184
|
this.indexes = /* @__PURE__ */ new Map();
|
|
7185
|
+
this.dirtyIndexes = /* @__PURE__ */ new Set();
|
|
7186
|
+
this.deletedIndexes = /* @__PURE__ */ new Set();
|
|
6870
7187
|
}
|
|
6871
7188
|
async onInstall() {
|
|
6872
7189
|
const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
|
|
@@ -6880,7 +7197,11 @@ class FullTextPlugin extends Plugin {
|
|
|
6880
7197
|
// Array of record IDs containing this word
|
|
6881
7198
|
count: "number|required",
|
|
6882
7199
|
lastUpdated: "string|required"
|
|
6883
|
-
}
|
|
7200
|
+
},
|
|
7201
|
+
partitions: {
|
|
7202
|
+
byResource: { fields: { resourceName: "string" } }
|
|
7203
|
+
},
|
|
7204
|
+
behavior: "body-overflow"
|
|
6884
7205
|
}));
|
|
6885
7206
|
this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
|
|
6886
7207
|
await this.loadIndexes();
|
|
@@ -6909,22 +7230,53 @@ class FullTextPlugin extends Plugin {
|
|
|
6909
7230
|
async saveIndexes() {
|
|
6910
7231
|
if (!this.indexResource) return;
|
|
6911
7232
|
const [ok, err] = await tryFn(async () => {
|
|
6912
|
-
const
|
|
6913
|
-
|
|
6914
|
-
await
|
|
7233
|
+
for (const key of this.deletedIndexes) {
|
|
7234
|
+
const [resourceName] = key.split(":");
|
|
7235
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7236
|
+
() => this.indexResource.query({ resourceName })
|
|
7237
|
+
);
|
|
7238
|
+
if (queryOk && results) {
|
|
7239
|
+
for (const index of results) {
|
|
7240
|
+
const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
|
|
7241
|
+
if (indexKey === key) {
|
|
7242
|
+
await this.indexResource.delete(index.id);
|
|
7243
|
+
}
|
|
7244
|
+
}
|
|
7245
|
+
}
|
|
6915
7246
|
}
|
|
6916
|
-
for (const
|
|
7247
|
+
for (const key of this.dirtyIndexes) {
|
|
6917
7248
|
const [resourceName, fieldName, word] = key.split(":");
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
7249
|
+
const data = this.indexes.get(key);
|
|
7250
|
+
if (!data) continue;
|
|
7251
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7252
|
+
() => this.indexResource.query({ resourceName })
|
|
7253
|
+
);
|
|
7254
|
+
let existingRecord = null;
|
|
7255
|
+
if (queryOk && results) {
|
|
7256
|
+
existingRecord = results.find(
|
|
7257
|
+
(index) => index.resourceName === resourceName && index.fieldName === fieldName && index.word === word
|
|
7258
|
+
);
|
|
7259
|
+
}
|
|
7260
|
+
if (existingRecord) {
|
|
7261
|
+
await this.indexResource.update(existingRecord.id, {
|
|
7262
|
+
recordIds: data.recordIds,
|
|
7263
|
+
count: data.count,
|
|
7264
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7265
|
+
});
|
|
7266
|
+
} else {
|
|
7267
|
+
await this.indexResource.insert({
|
|
7268
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
7269
|
+
resourceName,
|
|
7270
|
+
fieldName,
|
|
7271
|
+
word,
|
|
7272
|
+
recordIds: data.recordIds,
|
|
7273
|
+
count: data.count,
|
|
7274
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7275
|
+
});
|
|
7276
|
+
}
|
|
6927
7277
|
}
|
|
7278
|
+
this.dirtyIndexes.clear();
|
|
7279
|
+
this.deletedIndexes.clear();
|
|
6928
7280
|
});
|
|
6929
7281
|
}
|
|
6930
7282
|
installDatabaseHooks() {
|
|
@@ -7019,6 +7371,7 @@ class FullTextPlugin extends Plugin {
|
|
|
7019
7371
|
existing.count = existing.recordIds.length;
|
|
7020
7372
|
}
|
|
7021
7373
|
this.indexes.set(key, existing);
|
|
7374
|
+
this.dirtyIndexes.add(key);
|
|
7022
7375
|
}
|
|
7023
7376
|
}
|
|
7024
7377
|
}
|
|
@@ -7031,8 +7384,10 @@ class FullTextPlugin extends Plugin {
|
|
|
7031
7384
|
data.count = data.recordIds.length;
|
|
7032
7385
|
if (data.recordIds.length === 0) {
|
|
7033
7386
|
this.indexes.delete(key);
|
|
7387
|
+
this.deletedIndexes.add(key);
|
|
7034
7388
|
} else {
|
|
7035
7389
|
this.indexes.set(key, data);
|
|
7390
|
+
this.dirtyIndexes.add(key);
|
|
7036
7391
|
}
|
|
7037
7392
|
}
|
|
7038
7393
|
}
|
|
@@ -7264,8 +7619,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7264
7619
|
errors: "number|required",
|
|
7265
7620
|
avgTime: "number|required",
|
|
7266
7621
|
timestamp: "string|required",
|
|
7267
|
-
metadata: "json"
|
|
7268
|
-
|
|
7622
|
+
metadata: "json",
|
|
7623
|
+
createdAt: "string|required"
|
|
7624
|
+
// YYYY-MM-DD for partitioning
|
|
7625
|
+
},
|
|
7626
|
+
partitions: {
|
|
7627
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7628
|
+
},
|
|
7629
|
+
behavior: "body-overflow"
|
|
7269
7630
|
}));
|
|
7270
7631
|
this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
|
|
7271
7632
|
const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7276,8 +7637,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7276
7637
|
operation: "string|required",
|
|
7277
7638
|
error: "string|required",
|
|
7278
7639
|
timestamp: "string|required",
|
|
7279
|
-
metadata: "json"
|
|
7280
|
-
|
|
7640
|
+
metadata: "json",
|
|
7641
|
+
createdAt: "string|required"
|
|
7642
|
+
// YYYY-MM-DD for partitioning
|
|
7643
|
+
},
|
|
7644
|
+
partitions: {
|
|
7645
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7646
|
+
},
|
|
7647
|
+
behavior: "body-overflow"
|
|
7281
7648
|
}));
|
|
7282
7649
|
this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
|
|
7283
7650
|
const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7288,8 +7655,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7288
7655
|
operation: "string|required",
|
|
7289
7656
|
duration: "number|required",
|
|
7290
7657
|
timestamp: "string|required",
|
|
7291
|
-
metadata: "json"
|
|
7292
|
-
|
|
7658
|
+
metadata: "json",
|
|
7659
|
+
createdAt: "string|required"
|
|
7660
|
+
// YYYY-MM-DD for partitioning
|
|
7661
|
+
},
|
|
7662
|
+
partitions: {
|
|
7663
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7664
|
+
},
|
|
7665
|
+
behavior: "body-overflow"
|
|
7293
7666
|
}));
|
|
7294
7667
|
this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
|
|
7295
7668
|
});
|
|
@@ -7510,6 +7883,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7510
7883
|
errorMetadata = { error: "true" };
|
|
7511
7884
|
resourceMetadata = { resource: "true" };
|
|
7512
7885
|
}
|
|
7886
|
+
const now = /* @__PURE__ */ new Date();
|
|
7887
|
+
const createdAt = now.toISOString().slice(0, 10);
|
|
7513
7888
|
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
7514
7889
|
if (data.count > 0) {
|
|
7515
7890
|
await this.metricsResource.insert({
|
|
@@ -7521,7 +7896,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7521
7896
|
totalTime: data.totalTime,
|
|
7522
7897
|
errors: data.errors,
|
|
7523
7898
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7524
|
-
timestamp:
|
|
7899
|
+
timestamp: now.toISOString(),
|
|
7900
|
+
createdAt,
|
|
7525
7901
|
metadata
|
|
7526
7902
|
});
|
|
7527
7903
|
}
|
|
@@ -7538,7 +7914,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7538
7914
|
totalTime: data.totalTime,
|
|
7539
7915
|
errors: data.errors,
|
|
7540
7916
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7541
|
-
timestamp:
|
|
7917
|
+
timestamp: now.toISOString(),
|
|
7918
|
+
createdAt,
|
|
7542
7919
|
metadata: resourceMetadata
|
|
7543
7920
|
});
|
|
7544
7921
|
}
|
|
@@ -7552,6 +7929,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7552
7929
|
operation: perf.operation,
|
|
7553
7930
|
duration: perf.duration,
|
|
7554
7931
|
timestamp: perf.timestamp,
|
|
7932
|
+
createdAt: perf.timestamp.slice(0, 10),
|
|
7933
|
+
// YYYY-MM-DD from timestamp
|
|
7555
7934
|
metadata: perfMetadata
|
|
7556
7935
|
});
|
|
7557
7936
|
}
|
|
@@ -7565,6 +7944,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7565
7944
|
error: error.error,
|
|
7566
7945
|
stack: error.stack,
|
|
7567
7946
|
timestamp: error.timestamp,
|
|
7947
|
+
createdAt: error.timestamp.slice(0, 10),
|
|
7948
|
+
// YYYY-MM-DD from timestamp
|
|
7568
7949
|
metadata: errorMetadata
|
|
7569
7950
|
});
|
|
7570
7951
|
}
|
|
@@ -7696,22 +8077,47 @@ class MetricsPlugin extends Plugin {
|
|
|
7696
8077
|
async cleanupOldData() {
|
|
7697
8078
|
const cutoffDate = /* @__PURE__ */ new Date();
|
|
7698
8079
|
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
8080
|
+
cutoffDate.toISOString().slice(0, 10);
|
|
8081
|
+
const datesToDelete = [];
|
|
8082
|
+
const startDate = new Date(cutoffDate);
|
|
8083
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
8084
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
8085
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
8086
|
+
}
|
|
7699
8087
|
if (this.metricsResource) {
|
|
7700
|
-
const
|
|
7701
|
-
|
|
7702
|
-
|
|
8088
|
+
for (const dateStr of datesToDelete) {
|
|
8089
|
+
const [ok, err, oldMetrics] = await tryFn(
|
|
8090
|
+
() => this.metricsResource.query({ createdAt: dateStr })
|
|
8091
|
+
);
|
|
8092
|
+
if (ok && oldMetrics) {
|
|
8093
|
+
for (const metric of oldMetrics) {
|
|
8094
|
+
await tryFn(() => this.metricsResource.delete(metric.id));
|
|
8095
|
+
}
|
|
8096
|
+
}
|
|
7703
8097
|
}
|
|
7704
8098
|
}
|
|
7705
8099
|
if (this.errorsResource) {
|
|
7706
|
-
const
|
|
7707
|
-
|
|
7708
|
-
|
|
8100
|
+
for (const dateStr of datesToDelete) {
|
|
8101
|
+
const [ok, err, oldErrors] = await tryFn(
|
|
8102
|
+
() => this.errorsResource.query({ createdAt: dateStr })
|
|
8103
|
+
);
|
|
8104
|
+
if (ok && oldErrors) {
|
|
8105
|
+
for (const error of oldErrors) {
|
|
8106
|
+
await tryFn(() => this.errorsResource.delete(error.id));
|
|
8107
|
+
}
|
|
8108
|
+
}
|
|
7709
8109
|
}
|
|
7710
8110
|
}
|
|
7711
8111
|
if (this.performanceResource) {
|
|
7712
|
-
const
|
|
7713
|
-
|
|
7714
|
-
|
|
8112
|
+
for (const dateStr of datesToDelete) {
|
|
8113
|
+
const [ok, err, oldPerformance] = await tryFn(
|
|
8114
|
+
() => this.performanceResource.query({ createdAt: dateStr })
|
|
8115
|
+
);
|
|
8116
|
+
if (ok && oldPerformance) {
|
|
8117
|
+
for (const perf of oldPerformance) {
|
|
8118
|
+
await tryFn(() => this.performanceResource.delete(perf.id));
|
|
8119
|
+
}
|
|
8120
|
+
}
|
|
7715
8121
|
}
|
|
7716
8122
|
}
|
|
7717
8123
|
}
|
|
@@ -12782,7 +13188,7 @@ class Database extends EventEmitter {
|
|
|
12782
13188
|
this.id = idGenerator(7);
|
|
12783
13189
|
this.version = "1";
|
|
12784
13190
|
this.s3dbVersion = (() => {
|
|
12785
|
-
const [ok, err, version] = tryFn(() => true ? "11.0.
|
|
13191
|
+
const [ok, err, version] = tryFn(() => true ? "11.0.3" : "latest");
|
|
12786
13192
|
return ok ? version : "latest";
|
|
12787
13193
|
})();
|
|
12788
13194
|
this.resources = {};
|
|
@@ -15091,28 +15497,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15091
15497
|
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
15092
15498
|
}
|
|
15093
15499
|
this.queueResource = this.database.resources[queueName];
|
|
15094
|
-
const lockName = `${this.config.resource}_locks`;
|
|
15095
|
-
const [okLock, errLock] = await tryFn(
|
|
15096
|
-
() => this.database.createResource({
|
|
15097
|
-
name: lockName,
|
|
15098
|
-
attributes: {
|
|
15099
|
-
id: "string|required",
|
|
15100
|
-
workerId: "string|required",
|
|
15101
|
-
timestamp: "number|required",
|
|
15102
|
-
ttl: "number|default:5000"
|
|
15103
|
-
},
|
|
15104
|
-
behavior: "body-overflow",
|
|
15105
|
-
timestamps: false
|
|
15106
|
-
})
|
|
15107
|
-
);
|
|
15108
|
-
if (okLock || this.database.resources[lockName]) {
|
|
15109
|
-
this.lockResource = this.database.resources[lockName];
|
|
15110
|
-
} else {
|
|
15111
|
-
this.lockResource = null;
|
|
15112
|
-
if (this.config.verbose) {
|
|
15113
|
-
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
15114
|
-
}
|
|
15115
|
-
}
|
|
15116
15500
|
this.addHelperMethods();
|
|
15117
15501
|
if (this.config.deadLetterResource) {
|
|
15118
15502
|
await this.createDeadLetterResource();
|
|
@@ -15183,13 +15567,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15183
15567
|
}
|
|
15184
15568
|
}
|
|
15185
15569
|
}, 5e3);
|
|
15186
|
-
this.lockCleanupInterval = setInterval(() => {
|
|
15187
|
-
this.cleanupStaleLocks().catch((err) => {
|
|
15188
|
-
if (this.config.verbose) {
|
|
15189
|
-
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
15190
|
-
}
|
|
15191
|
-
});
|
|
15192
|
-
}, 1e4);
|
|
15193
15570
|
for (let i = 0; i < concurrency; i++) {
|
|
15194
15571
|
const worker = this.createWorker(messageHandler, i);
|
|
15195
15572
|
this.workers.push(worker);
|
|
@@ -15206,10 +15583,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15206
15583
|
clearInterval(this.cacheCleanupInterval);
|
|
15207
15584
|
this.cacheCleanupInterval = null;
|
|
15208
15585
|
}
|
|
15209
|
-
if (this.lockCleanupInterval) {
|
|
15210
|
-
clearInterval(this.lockCleanupInterval);
|
|
15211
|
-
this.lockCleanupInterval = null;
|
|
15212
|
-
}
|
|
15213
15586
|
await Promise.all(this.workers);
|
|
15214
15587
|
this.workers = [];
|
|
15215
15588
|
this.processedCache.clear();
|
|
@@ -15260,48 +15633,21 @@ class S3QueuePlugin extends Plugin {
|
|
|
15260
15633
|
return null;
|
|
15261
15634
|
}
|
|
15262
15635
|
/**
|
|
15263
|
-
* Acquire a distributed lock using
|
|
15636
|
+
* Acquire a distributed lock using PluginStorage TTL
|
|
15264
15637
|
* This ensures only one worker can claim a message at a time
|
|
15265
|
-
*
|
|
15266
|
-
* Uses a two-step process:
|
|
15267
|
-
* 1. Create lock resource (similar to queue resource) if not exists
|
|
15268
|
-
* 2. Try to claim lock using ETag-based conditional update
|
|
15269
15638
|
*/
|
|
15270
15639
|
async acquireLock(messageId) {
|
|
15271
|
-
|
|
15272
|
-
|
|
15273
|
-
}
|
|
15274
|
-
const lockId = `lock-${messageId}`;
|
|
15275
|
-
const now = Date.now();
|
|
15640
|
+
const storage = this.getStorage();
|
|
15641
|
+
const lockKey = `msg-${messageId}`;
|
|
15276
15642
|
try {
|
|
15277
|
-
const
|
|
15278
|
-
|
|
15279
|
-
|
|
15280
|
-
|
|
15281
|
-
|
|
15282
|
-
|
|
15283
|
-
|
|
15284
|
-
|
|
15285
|
-
const [ok, err, result] = await tryFn(
|
|
15286
|
-
() => this.lockResource.updateConditional(lockId, {
|
|
15287
|
-
workerId: this.workerId,
|
|
15288
|
-
timestamp: now,
|
|
15289
|
-
ttl: 5e3
|
|
15290
|
-
}, {
|
|
15291
|
-
ifMatch: existingLock._etag
|
|
15292
|
-
})
|
|
15293
|
-
);
|
|
15294
|
-
return ok && result.success;
|
|
15295
|
-
}
|
|
15296
|
-
const [okCreate, errCreate] = await tryFn(
|
|
15297
|
-
() => this.lockResource.insert({
|
|
15298
|
-
id: lockId,
|
|
15299
|
-
workerId: this.workerId,
|
|
15300
|
-
timestamp: now,
|
|
15301
|
-
ttl: 5e3
|
|
15302
|
-
})
|
|
15303
|
-
);
|
|
15304
|
-
return okCreate;
|
|
15643
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
15644
|
+
ttl: 5,
|
|
15645
|
+
// 5 seconds
|
|
15646
|
+
timeout: 0,
|
|
15647
|
+
// Don't wait if locked
|
|
15648
|
+
workerId: this.workerId
|
|
15649
|
+
});
|
|
15650
|
+
return lock !== null;
|
|
15305
15651
|
} catch (error) {
|
|
15306
15652
|
if (this.config.verbose) {
|
|
15307
15653
|
console.log(`[acquireLock] Error: ${error.message}`);
|
|
@@ -15310,15 +15656,13 @@ class S3QueuePlugin extends Plugin {
|
|
|
15310
15656
|
}
|
|
15311
15657
|
}
|
|
15312
15658
|
/**
|
|
15313
|
-
* Release a distributed lock
|
|
15659
|
+
* Release a distributed lock via PluginStorage
|
|
15314
15660
|
*/
|
|
15315
15661
|
async releaseLock(messageId) {
|
|
15316
|
-
|
|
15317
|
-
|
|
15318
|
-
}
|
|
15319
|
-
const lockId = `lock-${messageId}`;
|
|
15662
|
+
const storage = this.getStorage();
|
|
15663
|
+
const lockKey = `msg-${messageId}`;
|
|
15320
15664
|
try {
|
|
15321
|
-
await
|
|
15665
|
+
await storage.releaseLock(lockKey);
|
|
15322
15666
|
} catch (error) {
|
|
15323
15667
|
if (this.config.verbose) {
|
|
15324
15668
|
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
@@ -15326,30 +15670,11 @@ class S3QueuePlugin extends Plugin {
|
|
|
15326
15670
|
}
|
|
15327
15671
|
}
|
|
15328
15672
|
/**
|
|
15329
|
-
* Clean up stale locks
|
|
15330
|
-
*
|
|
15673
|
+
* Clean up stale locks - NO LONGER NEEDED
|
|
15674
|
+
* TTL handles automatic expiration, no manual cleanup required
|
|
15331
15675
|
*/
|
|
15332
15676
|
async cleanupStaleLocks() {
|
|
15333
|
-
|
|
15334
|
-
return;
|
|
15335
|
-
}
|
|
15336
|
-
const now = Date.now();
|
|
15337
|
-
try {
|
|
15338
|
-
const locks = await this.lockResource.list();
|
|
15339
|
-
for (const lock of locks) {
|
|
15340
|
-
const lockAge = now - lock.timestamp;
|
|
15341
|
-
if (lockAge > lock.ttl) {
|
|
15342
|
-
await this.lockResource.delete(lock.id);
|
|
15343
|
-
if (this.config.verbose) {
|
|
15344
|
-
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
15345
|
-
}
|
|
15346
|
-
}
|
|
15347
|
-
}
|
|
15348
|
-
} catch (error) {
|
|
15349
|
-
if (this.config.verbose) {
|
|
15350
|
-
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
15351
|
-
}
|
|
15352
|
-
}
|
|
15677
|
+
return;
|
|
15353
15678
|
}
|
|
15354
15679
|
async attemptClaim(msg) {
|
|
15355
15680
|
const now = Date.now();
|
|
@@ -15573,7 +15898,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15573
15898
|
...options
|
|
15574
15899
|
};
|
|
15575
15900
|
this.database = null;
|
|
15576
|
-
this.lockResource = null;
|
|
15577
15901
|
this.jobs = /* @__PURE__ */ new Map();
|
|
15578
15902
|
this.activeJobs = /* @__PURE__ */ new Map();
|
|
15579
15903
|
this.timers = /* @__PURE__ */ new Map();
|
|
@@ -15612,7 +15936,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15612
15936
|
return true;
|
|
15613
15937
|
}
|
|
15614
15938
|
async onInstall() {
|
|
15615
|
-
await this._createLockResource();
|
|
15616
15939
|
if (this.config.persistJobs) {
|
|
15617
15940
|
await this._createJobHistoryResource();
|
|
15618
15941
|
}
|
|
@@ -15641,25 +15964,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15641
15964
|
await this._startScheduling();
|
|
15642
15965
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
15643
15966
|
}
|
|
15644
|
-
async _createLockResource() {
|
|
15645
|
-
const [ok, err, lockResource] = await tryFn(
|
|
15646
|
-
() => this.database.createResource({
|
|
15647
|
-
name: "plg_scheduler_job_locks",
|
|
15648
|
-
attributes: {
|
|
15649
|
-
id: "string|required",
|
|
15650
|
-
jobName: "string|required",
|
|
15651
|
-
lockedAt: "number|required",
|
|
15652
|
-
instanceId: "string|optional"
|
|
15653
|
-
},
|
|
15654
|
-
behavior: "body-only",
|
|
15655
|
-
timestamps: false
|
|
15656
|
-
})
|
|
15657
|
-
);
|
|
15658
|
-
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
15659
|
-
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
15660
|
-
}
|
|
15661
|
-
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
15662
|
-
}
|
|
15663
15967
|
async _createJobHistoryResource() {
|
|
15664
15968
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
15665
15969
|
name: this.config.jobHistoryResource,
|
|
@@ -15767,16 +16071,16 @@ class SchedulerPlugin extends Plugin {
|
|
|
15767
16071
|
return;
|
|
15768
16072
|
}
|
|
15769
16073
|
this.activeJobs.set(jobName, "acquiring-lock");
|
|
15770
|
-
const
|
|
15771
|
-
const
|
|
15772
|
-
|
|
15773
|
-
|
|
15774
|
-
|
|
15775
|
-
|
|
15776
|
-
|
|
15777
|
-
|
|
15778
|
-
);
|
|
15779
|
-
if (!
|
|
16074
|
+
const storage = this.getStorage();
|
|
16075
|
+
const lockKey = `job-${jobName}`;
|
|
16076
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
16077
|
+
ttl: Math.ceil(job.timeout / 1e3) + 60,
|
|
16078
|
+
// Job timeout + 60 seconds buffer
|
|
16079
|
+
timeout: 0,
|
|
16080
|
+
// Don't wait if locked
|
|
16081
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
16082
|
+
});
|
|
16083
|
+
if (!lock) {
|
|
15780
16084
|
if (this.config.verbose) {
|
|
15781
16085
|
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
15782
16086
|
}
|
|
@@ -15880,7 +16184,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
15880
16184
|
throw lastError;
|
|
15881
16185
|
}
|
|
15882
16186
|
} finally {
|
|
15883
|
-
await tryFn(() =>
|
|
16187
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
15884
16188
|
}
|
|
15885
16189
|
}
|
|
15886
16190
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|