s3db.js 11.0.1 → 11.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +621 -319
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +621 -319
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/concerns/plugin-storage.js +274 -9
- package/src/plugins/audit.plugin.js +94 -18
- package/src/plugins/eventual-consistency/analytics.js +136 -20
- package/src/plugins/eventual-consistency/config.js +4 -2
- package/src/plugins/eventual-consistency/consolidation.js +35 -39
- package/src/plugins/eventual-consistency/garbage-collection.js +11 -13
- package/src/plugins/eventual-consistency/index.js +28 -19
- package/src/plugins/eventual-consistency/install.js +9 -26
- package/src/plugins/eventual-consistency/partitions.js +5 -0
- package/src/plugins/eventual-consistency/transactions.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +36 -1
- package/src/plugins/fulltext.plugin.js +76 -22
- package/src/plugins/metrics.plugin.js +70 -20
- package/src/plugins/s3-queue.plugin.js +21 -120
- package/src/plugins/scheduler.plugin.js +11 -37
- package/dist/s3db-cli.js +0 -54741
package/dist/s3db.cjs.js
CHANGED
|
@@ -837,18 +837,23 @@ class PluginStorage {
|
|
|
837
837
|
return `plugin=${this.pluginSlug}/${parts.join("/")}`;
|
|
838
838
|
}
|
|
839
839
|
/**
|
|
840
|
-
* Save data with metadata encoding and
|
|
840
|
+
* Save data with metadata encoding, behavior support, and optional TTL
|
|
841
841
|
*
|
|
842
842
|
* @param {string} key - S3 key
|
|
843
843
|
* @param {Object} data - Data to save
|
|
844
844
|
* @param {Object} options - Options
|
|
845
|
+
* @param {number} options.ttl - Time-to-live in seconds (optional)
|
|
845
846
|
* @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
|
|
846
847
|
* @param {string} options.contentType - Content type (default: application/json)
|
|
847
848
|
* @returns {Promise<void>}
|
|
848
849
|
*/
|
|
849
|
-
async
|
|
850
|
-
const { behavior = "body-overflow", contentType = "application/json" } = options;
|
|
851
|
-
const
|
|
850
|
+
async set(key, data, options = {}) {
|
|
851
|
+
const { ttl, behavior = "body-overflow", contentType = "application/json" } = options;
|
|
852
|
+
const dataToSave = { ...data };
|
|
853
|
+
if (ttl && typeof ttl === "number" && ttl > 0) {
|
|
854
|
+
dataToSave._expiresAt = Date.now() + ttl * 1e3;
|
|
855
|
+
}
|
|
856
|
+
const { metadata, body } = this._applyBehavior(dataToSave, behavior);
|
|
852
857
|
const putParams = {
|
|
853
858
|
key,
|
|
854
859
|
metadata,
|
|
@@ -859,14 +864,21 @@ class PluginStorage {
|
|
|
859
864
|
}
|
|
860
865
|
const [ok, err] = await tryFn(() => this.client.putObject(putParams));
|
|
861
866
|
if (!ok) {
|
|
862
|
-
throw new Error(`PluginStorage.
|
|
867
|
+
throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
|
|
863
868
|
}
|
|
864
869
|
}
|
|
865
870
|
/**
|
|
866
|
-
*
|
|
871
|
+
* Alias for set() to maintain backward compatibility
|
|
872
|
+
* @deprecated Use set() instead
|
|
873
|
+
*/
|
|
874
|
+
async put(key, data, options = {}) {
|
|
875
|
+
return this.set(key, data, options);
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Get data with automatic metadata decoding and TTL check
|
|
867
879
|
*
|
|
868
880
|
* @param {string} key - S3 key
|
|
869
|
-
* @returns {Promise<Object|null>} Data or null if not found
|
|
881
|
+
* @returns {Promise<Object|null>} Data or null if not found/expired
|
|
870
882
|
*/
|
|
871
883
|
async get(key) {
|
|
872
884
|
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
@@ -878,18 +890,28 @@ class PluginStorage {
|
|
|
878
890
|
}
|
|
879
891
|
const metadata = response.Metadata || {};
|
|
880
892
|
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
893
|
+
let data = parsedMetadata;
|
|
881
894
|
if (response.Body) {
|
|
882
895
|
try {
|
|
883
896
|
const bodyContent = await response.Body.transformToString();
|
|
884
897
|
if (bodyContent && bodyContent.trim()) {
|
|
885
898
|
const body = JSON.parse(bodyContent);
|
|
886
|
-
|
|
899
|
+
data = { ...parsedMetadata, ...body };
|
|
887
900
|
}
|
|
888
901
|
} catch (parseErr) {
|
|
889
902
|
throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
|
|
890
903
|
}
|
|
891
904
|
}
|
|
892
|
-
|
|
905
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
906
|
+
if (expiresAt) {
|
|
907
|
+
if (Date.now() > expiresAt) {
|
|
908
|
+
await this.delete(key);
|
|
909
|
+
return null;
|
|
910
|
+
}
|
|
911
|
+
delete data._expiresat;
|
|
912
|
+
delete data._expiresAt;
|
|
913
|
+
}
|
|
914
|
+
return data;
|
|
893
915
|
}
|
|
894
916
|
/**
|
|
895
917
|
* Parse metadata values back to their original types
|
|
@@ -972,6 +994,123 @@ class PluginStorage {
|
|
|
972
994
|
if (!keyPrefix) return keys;
|
|
973
995
|
return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
|
|
974
996
|
}
|
|
997
|
+
/**
|
|
998
|
+
* Check if a key exists (not expired)
|
|
999
|
+
*
|
|
1000
|
+
* @param {string} key - S3 key
|
|
1001
|
+
* @returns {Promise<boolean>} True if exists and not expired
|
|
1002
|
+
*/
|
|
1003
|
+
async has(key) {
|
|
1004
|
+
const data = await this.get(key);
|
|
1005
|
+
return data !== null;
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Check if a key is expired
|
|
1009
|
+
*
|
|
1010
|
+
* @param {string} key - S3 key
|
|
1011
|
+
* @returns {Promise<boolean>} True if expired or not found
|
|
1012
|
+
*/
|
|
1013
|
+
async isExpired(key) {
|
|
1014
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1015
|
+
if (!ok) {
|
|
1016
|
+
return true;
|
|
1017
|
+
}
|
|
1018
|
+
const metadata = response.Metadata || {};
|
|
1019
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1020
|
+
let data = parsedMetadata;
|
|
1021
|
+
if (response.Body) {
|
|
1022
|
+
try {
|
|
1023
|
+
const bodyContent = await response.Body.transformToString();
|
|
1024
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1025
|
+
const body = JSON.parse(bodyContent);
|
|
1026
|
+
data = { ...parsedMetadata, ...body };
|
|
1027
|
+
}
|
|
1028
|
+
} catch {
|
|
1029
|
+
return true;
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1033
|
+
if (!expiresAt) {
|
|
1034
|
+
return false;
|
|
1035
|
+
}
|
|
1036
|
+
return Date.now() > expiresAt;
|
|
1037
|
+
}
|
|
1038
|
+
/**
|
|
1039
|
+
* Get remaining TTL in seconds
|
|
1040
|
+
*
|
|
1041
|
+
* @param {string} key - S3 key
|
|
1042
|
+
* @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
|
|
1043
|
+
*/
|
|
1044
|
+
async getTTL(key) {
|
|
1045
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1046
|
+
if (!ok) {
|
|
1047
|
+
return null;
|
|
1048
|
+
}
|
|
1049
|
+
const metadata = response.Metadata || {};
|
|
1050
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1051
|
+
let data = parsedMetadata;
|
|
1052
|
+
if (response.Body) {
|
|
1053
|
+
try {
|
|
1054
|
+
const bodyContent = await response.Body.transformToString();
|
|
1055
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1056
|
+
const body = JSON.parse(bodyContent);
|
|
1057
|
+
data = { ...parsedMetadata, ...body };
|
|
1058
|
+
}
|
|
1059
|
+
} catch {
|
|
1060
|
+
return null;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1064
|
+
if (!expiresAt) {
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
const remaining = Math.max(0, expiresAt - Date.now());
|
|
1068
|
+
return Math.floor(remaining / 1e3);
|
|
1069
|
+
}
|
|
1070
|
+
/**
|
|
1071
|
+
* Extend TTL by adding additional seconds
|
|
1072
|
+
*
|
|
1073
|
+
* @param {string} key - S3 key
|
|
1074
|
+
* @param {number} additionalSeconds - Seconds to add to current TTL
|
|
1075
|
+
* @returns {Promise<boolean>} True if extended, false if not found or no TTL
|
|
1076
|
+
*/
|
|
1077
|
+
async touch(key, additionalSeconds) {
|
|
1078
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1079
|
+
if (!ok) {
|
|
1080
|
+
return false;
|
|
1081
|
+
}
|
|
1082
|
+
const metadata = response.Metadata || {};
|
|
1083
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1084
|
+
let data = parsedMetadata;
|
|
1085
|
+
if (response.Body) {
|
|
1086
|
+
try {
|
|
1087
|
+
const bodyContent = await response.Body.transformToString();
|
|
1088
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1089
|
+
const body = JSON.parse(bodyContent);
|
|
1090
|
+
data = { ...parsedMetadata, ...body };
|
|
1091
|
+
}
|
|
1092
|
+
} catch {
|
|
1093
|
+
return false;
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1097
|
+
if (!expiresAt) {
|
|
1098
|
+
return false;
|
|
1099
|
+
}
|
|
1100
|
+
data._expiresAt = expiresAt + additionalSeconds * 1e3;
|
|
1101
|
+
delete data._expiresat;
|
|
1102
|
+
const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, "body-overflow");
|
|
1103
|
+
const putParams = {
|
|
1104
|
+
key,
|
|
1105
|
+
metadata: newMetadata,
|
|
1106
|
+
contentType: "application/json"
|
|
1107
|
+
};
|
|
1108
|
+
if (newBody !== null) {
|
|
1109
|
+
putParams.body = JSON.stringify(newBody);
|
|
1110
|
+
}
|
|
1111
|
+
const [putOk] = await tryFn(() => this.client.putObject(putParams));
|
|
1112
|
+
return putOk;
|
|
1113
|
+
}
|
|
975
1114
|
/**
|
|
976
1115
|
* Delete a single object
|
|
977
1116
|
*
|
|
@@ -1049,6 +1188,78 @@ class PluginStorage {
|
|
|
1049
1188
|
}
|
|
1050
1189
|
return results;
|
|
1051
1190
|
}
|
|
1191
|
+
/**
|
|
1192
|
+
* Acquire a distributed lock with TTL and retry logic
|
|
1193
|
+
*
|
|
1194
|
+
* @param {string} lockName - Lock identifier
|
|
1195
|
+
* @param {Object} options - Lock options
|
|
1196
|
+
* @param {number} options.ttl - Lock TTL in seconds (default: 30)
|
|
1197
|
+
* @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
|
|
1198
|
+
* @param {string} options.workerId - Worker identifier (default: 'unknown')
|
|
1199
|
+
* @returns {Promise<Object|null>} Lock object or null if couldn't acquire
|
|
1200
|
+
*/
|
|
1201
|
+
async acquireLock(lockName, options = {}) {
|
|
1202
|
+
const { ttl = 30, timeout = 0, workerId = "unknown" } = options;
|
|
1203
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1204
|
+
const startTime = Date.now();
|
|
1205
|
+
while (true) {
|
|
1206
|
+
const existing = await this.get(key);
|
|
1207
|
+
if (!existing) {
|
|
1208
|
+
await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
|
|
1209
|
+
return { key, workerId };
|
|
1210
|
+
}
|
|
1211
|
+
if (Date.now() - startTime >= timeout) {
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Release a distributed lock
|
|
1219
|
+
*
|
|
1220
|
+
* @param {string} lockName - Lock identifier
|
|
1221
|
+
* @returns {Promise<void>}
|
|
1222
|
+
*/
|
|
1223
|
+
async releaseLock(lockName) {
|
|
1224
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1225
|
+
await this.delete(key);
|
|
1226
|
+
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Check if a lock is currently held
|
|
1229
|
+
*
|
|
1230
|
+
* @param {string} lockName - Lock identifier
|
|
1231
|
+
* @returns {Promise<boolean>} True if locked
|
|
1232
|
+
*/
|
|
1233
|
+
async isLocked(lockName) {
|
|
1234
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1235
|
+
const lock = await this.get(key);
|
|
1236
|
+
return lock !== null;
|
|
1237
|
+
}
|
|
1238
|
+
/**
|
|
1239
|
+
* Increment a counter value
|
|
1240
|
+
*
|
|
1241
|
+
* @param {string} key - S3 key
|
|
1242
|
+
* @param {number} amount - Amount to increment (default: 1)
|
|
1243
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1244
|
+
* @returns {Promise<number>} New value
|
|
1245
|
+
*/
|
|
1246
|
+
async increment(key, amount = 1, options = {}) {
|
|
1247
|
+
const data = await this.get(key);
|
|
1248
|
+
const value = (data?.value || 0) + amount;
|
|
1249
|
+
await this.set(key, { value }, options);
|
|
1250
|
+
return value;
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* Decrement a counter value
|
|
1254
|
+
*
|
|
1255
|
+
* @param {string} key - S3 key
|
|
1256
|
+
* @param {number} amount - Amount to decrement (default: 1)
|
|
1257
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1258
|
+
* @returns {Promise<number>} New value
|
|
1259
|
+
*/
|
|
1260
|
+
async decrement(key, amount = 1, options = {}) {
|
|
1261
|
+
return this.increment(key, -amount, options);
|
|
1262
|
+
}
|
|
1052
1263
|
/**
|
|
1053
1264
|
* Apply behavior to split data between metadata and body
|
|
1054
1265
|
*
|
|
@@ -1352,12 +1563,18 @@ class AuditPlugin extends Plugin {
|
|
|
1352
1563
|
recordId: "string|required",
|
|
1353
1564
|
userId: "string|optional",
|
|
1354
1565
|
timestamp: "string|required",
|
|
1566
|
+
createdAt: "string|required",
|
|
1567
|
+
// YYYY-MM-DD for partitioning
|
|
1355
1568
|
oldData: "string|optional",
|
|
1356
1569
|
newData: "string|optional",
|
|
1357
1570
|
partition: "string|optional",
|
|
1358
1571
|
partitionValues: "string|optional",
|
|
1359
1572
|
metadata: "string|optional"
|
|
1360
1573
|
},
|
|
1574
|
+
partitions: {
|
|
1575
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } },
|
|
1576
|
+
byResource: { fields: { resourceName: "string" } }
|
|
1577
|
+
},
|
|
1361
1578
|
behavior: "body-overflow"
|
|
1362
1579
|
}));
|
|
1363
1580
|
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
@@ -1461,10 +1678,13 @@ class AuditPlugin extends Plugin {
|
|
|
1461
1678
|
if (!this.auditResource) {
|
|
1462
1679
|
return;
|
|
1463
1680
|
}
|
|
1681
|
+
const now = /* @__PURE__ */ new Date();
|
|
1464
1682
|
const auditRecord = {
|
|
1465
1683
|
id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
|
|
1466
1684
|
userId: this.getCurrentUserId?.() || "system",
|
|
1467
|
-
timestamp:
|
|
1685
|
+
timestamp: now.toISOString(),
|
|
1686
|
+
createdAt: now.toISOString().slice(0, 10),
|
|
1687
|
+
// YYYY-MM-DD for partitioning
|
|
1468
1688
|
metadata: JSON.stringify({ source: "audit-plugin", version: "2.0" }),
|
|
1469
1689
|
resourceName: auditData.resourceName,
|
|
1470
1690
|
operation: auditData.operation,
|
|
@@ -1539,9 +1759,25 @@ class AuditPlugin extends Plugin {
|
|
|
1539
1759
|
async getAuditLogs(options = {}) {
|
|
1540
1760
|
if (!this.auditResource) return [];
|
|
1541
1761
|
const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
|
|
1542
|
-
const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
|
|
1543
1762
|
let items = [];
|
|
1544
|
-
if (
|
|
1763
|
+
if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
|
|
1764
|
+
const [ok, err, result] = await tryFn(
|
|
1765
|
+
() => this.auditResource.query({ resourceName }, { limit: limit + offset })
|
|
1766
|
+
);
|
|
1767
|
+
items = ok && result ? result : [];
|
|
1768
|
+
return items.slice(offset, offset + limit);
|
|
1769
|
+
} else if (startDate && !resourceName && !operation && !recordId && !partition) {
|
|
1770
|
+
const dates = this._generateDateRange(startDate, endDate);
|
|
1771
|
+
for (const date of dates) {
|
|
1772
|
+
const [ok, err, result] = await tryFn(
|
|
1773
|
+
() => this.auditResource.query({ createdAt: date })
|
|
1774
|
+
);
|
|
1775
|
+
if (ok && result) {
|
|
1776
|
+
items.push(...result);
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
return items.slice(offset, offset + limit);
|
|
1780
|
+
} else if (resourceName || operation || recordId || partition || startDate || endDate) {
|
|
1545
1781
|
const fetchSize = Math.min(1e4, Math.max(1e3, (limit + offset) * 20));
|
|
1546
1782
|
const result = await this.auditResource.list({ limit: fetchSize });
|
|
1547
1783
|
items = result || [];
|
|
@@ -1571,6 +1807,15 @@ class AuditPlugin extends Plugin {
|
|
|
1571
1807
|
return result.items || [];
|
|
1572
1808
|
}
|
|
1573
1809
|
}
|
|
1810
|
+
_generateDateRange(startDate, endDate) {
|
|
1811
|
+
const dates = [];
|
|
1812
|
+
const start = new Date(startDate);
|
|
1813
|
+
const end = endDate ? new Date(endDate) : /* @__PURE__ */ new Date();
|
|
1814
|
+
for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
|
|
1815
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
1816
|
+
}
|
|
1817
|
+
return dates;
|
|
1818
|
+
}
|
|
1574
1819
|
async getRecordHistory(resourceName, recordId) {
|
|
1575
1820
|
return await this.getAuditLogs({ resourceName, recordId });
|
|
1576
1821
|
}
|
|
@@ -1603,6 +1848,37 @@ class AuditPlugin extends Plugin {
|
|
|
1603
1848
|
}
|
|
1604
1849
|
return stats;
|
|
1605
1850
|
}
|
|
1851
|
+
/**
|
|
1852
|
+
* Clean up audit logs older than retention period
|
|
1853
|
+
* @param {number} retentionDays - Number of days to retain (default: 90)
|
|
1854
|
+
* @returns {Promise<number>} Number of records deleted
|
|
1855
|
+
*/
|
|
1856
|
+
async cleanupOldAudits(retentionDays = 90) {
|
|
1857
|
+
if (!this.auditResource) return 0;
|
|
1858
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
1859
|
+
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
|
1860
|
+
const datesToDelete = [];
|
|
1861
|
+
const startDate = new Date(cutoffDate);
|
|
1862
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
1863
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
1864
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
1865
|
+
}
|
|
1866
|
+
let deletedCount = 0;
|
|
1867
|
+
for (const dateStr of datesToDelete) {
|
|
1868
|
+
const [ok, err, oldAudits] = await tryFn(
|
|
1869
|
+
() => this.auditResource.query({ createdAt: dateStr })
|
|
1870
|
+
);
|
|
1871
|
+
if (ok && oldAudits) {
|
|
1872
|
+
for (const audit of oldAudits) {
|
|
1873
|
+
const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
|
|
1874
|
+
if (delOk) {
|
|
1875
|
+
deletedCount++;
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
return deletedCount;
|
|
1881
|
+
}
|
|
1606
1882
|
}
|
|
1607
1883
|
|
|
1608
1884
|
class BaseBackupDriver {
|
|
@@ -4838,6 +5114,8 @@ function createConfig(options, detectedTimezone) {
|
|
|
4838
5114
|
consolidationWindow: consolidation.window ?? 24,
|
|
4839
5115
|
autoConsolidate: consolidation.auto !== false,
|
|
4840
5116
|
mode: consolidation.mode || "async",
|
|
5117
|
+
// ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
|
|
5118
|
+
markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
|
|
4841
5119
|
// Late arrivals
|
|
4842
5120
|
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
4843
5121
|
// Batch transactions
|
|
@@ -4864,9 +5142,7 @@ function createConfig(options, detectedTimezone) {
|
|
|
4864
5142
|
deleteConsolidatedTransactions: checkpoints.deleteConsolidated !== false,
|
|
4865
5143
|
autoCheckpoint: checkpoints.auto !== false,
|
|
4866
5144
|
// Debug
|
|
4867
|
-
verbose: options.verbose
|
|
4868
|
-
// Default: true (can disable with verbose: false)
|
|
4869
|
-
debug: options.debug || false
|
|
5145
|
+
verbose: options.verbose || false
|
|
4870
5146
|
};
|
|
4871
5147
|
}
|
|
4872
5148
|
function validateResourcesConfig(resources) {
|
|
@@ -4943,6 +5219,21 @@ function getTimezoneOffset(timezone, verbose = false) {
|
|
|
4943
5219
|
return offsets[timezone] || 0;
|
|
4944
5220
|
}
|
|
4945
5221
|
}
|
|
5222
|
+
function getISOWeek(date) {
|
|
5223
|
+
const target = new Date(date.valueOf());
|
|
5224
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
5225
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
5226
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
5227
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
5228
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
5229
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
5230
|
+
}
|
|
5231
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
5232
|
+
return {
|
|
5233
|
+
year: target.getUTCFullYear(),
|
|
5234
|
+
week: weekNumber
|
|
5235
|
+
};
|
|
5236
|
+
}
|
|
4946
5237
|
function getCohortInfo(date, timezone, verbose = false) {
|
|
4947
5238
|
const offset = getTimezoneOffset(timezone, verbose);
|
|
4948
5239
|
const localDate = new Date(date.getTime() + offset);
|
|
@@ -4950,10 +5241,14 @@ function getCohortInfo(date, timezone, verbose = false) {
|
|
|
4950
5241
|
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4951
5242
|
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4952
5243
|
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
5244
|
+
const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
|
|
5245
|
+
const week = `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
4953
5246
|
return {
|
|
4954
5247
|
date: `${year}-${month}-${day}`,
|
|
4955
5248
|
hour: `${year}-${month}-${day}T${hour}`,
|
|
4956
5249
|
// ISO-like format for hour partition
|
|
5250
|
+
week,
|
|
5251
|
+
// ISO 8601 week format (e.g., '2025-W42')
|
|
4957
5252
|
month: `${year}-${month}`
|
|
4958
5253
|
};
|
|
4959
5254
|
}
|
|
@@ -5031,6 +5326,11 @@ function createPartitionConfig() {
|
|
|
5031
5326
|
cohortDate: "string"
|
|
5032
5327
|
}
|
|
5033
5328
|
},
|
|
5329
|
+
byWeek: {
|
|
5330
|
+
fields: {
|
|
5331
|
+
cohortWeek: "string"
|
|
5332
|
+
}
|
|
5333
|
+
},
|
|
5034
5334
|
byMonth: {
|
|
5035
5335
|
fields: {
|
|
5036
5336
|
cohortMonth: "string"
|
|
@@ -5070,6 +5370,7 @@ async function createTransaction(handler, data, config) {
|
|
|
5070
5370
|
timestamp: now.toISOString(),
|
|
5071
5371
|
cohortDate: cohortInfo.date,
|
|
5072
5372
|
cohortHour: cohortInfo.hour,
|
|
5373
|
+
cohortWeek: cohortInfo.week,
|
|
5073
5374
|
cohortMonth: cohortInfo.month,
|
|
5074
5375
|
source: data.source || "unknown",
|
|
5075
5376
|
applied: false
|
|
@@ -5110,50 +5411,6 @@ async function flushPendingTransactions(handler) {
|
|
|
5110
5411
|
}
|
|
5111
5412
|
}
|
|
5112
5413
|
|
|
5113
|
-
async function cleanupStaleLocks(lockResource, config) {
|
|
5114
|
-
const now = Date.now();
|
|
5115
|
-
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
5116
|
-
const cutoffTime = now - lockTimeoutMs;
|
|
5117
|
-
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
5118
|
-
const [lockAcquired] = await tryFn(
|
|
5119
|
-
() => lockResource.insert({
|
|
5120
|
-
id: cleanupLockId,
|
|
5121
|
-
lockedAt: Date.now(),
|
|
5122
|
-
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5123
|
-
})
|
|
5124
|
-
);
|
|
5125
|
-
if (!lockAcquired) {
|
|
5126
|
-
if (config.verbose) {
|
|
5127
|
-
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
5128
|
-
}
|
|
5129
|
-
return;
|
|
5130
|
-
}
|
|
5131
|
-
try {
|
|
5132
|
-
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
5133
|
-
if (!ok || !locks || locks.length === 0) return;
|
|
5134
|
-
const staleLocks = locks.filter(
|
|
5135
|
-
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
5136
|
-
);
|
|
5137
|
-
if (staleLocks.length === 0) return;
|
|
5138
|
-
if (config.verbose) {
|
|
5139
|
-
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
5140
|
-
}
|
|
5141
|
-
const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
5142
|
-
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
5143
|
-
return deleted;
|
|
5144
|
-
});
|
|
5145
|
-
if (errors && errors.length > 0 && config.verbose) {
|
|
5146
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5147
|
-
}
|
|
5148
|
-
} catch (error) {
|
|
5149
|
-
if (config.verbose) {
|
|
5150
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5151
|
-
}
|
|
5152
|
-
} finally {
|
|
5153
|
-
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
5154
|
-
}
|
|
5155
|
-
}
|
|
5156
|
-
|
|
5157
5414
|
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
5158
5415
|
const intervalMs = config.consolidationInterval * 1e3;
|
|
5159
5416
|
if (config.verbose) {
|
|
@@ -5250,17 +5507,15 @@ async function runConsolidation(transactionResource, consolidateRecordFn, emitFn
|
|
|
5250
5507
|
}
|
|
5251
5508
|
}
|
|
5252
5509
|
}
|
|
5253
|
-
async function consolidateRecord(originalId, transactionResource, targetResource,
|
|
5254
|
-
|
|
5255
|
-
const
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
);
|
|
5263
|
-
if (!lockAcquired) {
|
|
5510
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, storage, analyticsResource, updateAnalyticsFn, config) {
|
|
5511
|
+
const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
|
|
5512
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5513
|
+
ttl: config.lockTimeout || 30,
|
|
5514
|
+
timeout: 0,
|
|
5515
|
+
// Don't wait if locked
|
|
5516
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5517
|
+
});
|
|
5518
|
+
if (!lock) {
|
|
5264
5519
|
if (config.verbose) {
|
|
5265
5520
|
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
5266
5521
|
}
|
|
@@ -5405,7 +5660,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5405
5660
|
`[EventualConsistency] ${config.resource}.${config.field} - ${originalId}: ${currentValue} \u2192 ${consolidatedValue} (${consolidatedValue > currentValue ? "+" : ""}${consolidatedValue - currentValue})`
|
|
5406
5661
|
);
|
|
5407
5662
|
}
|
|
5408
|
-
if (config.
|
|
5663
|
+
if (config.verbose) {
|
|
5409
5664
|
console.log(
|
|
5410
5665
|
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5411
5666
|
originalId: '${originalId}',
|
|
@@ -5420,7 +5675,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5420
5675
|
[config.field]: consolidatedValue
|
|
5421
5676
|
})
|
|
5422
5677
|
);
|
|
5423
|
-
if (config.
|
|
5678
|
+
if (config.verbose) {
|
|
5424
5679
|
console.log(
|
|
5425
5680
|
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5426
5681
|
updateOk: ${updateOk},
|
|
@@ -5430,7 +5685,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5430
5685
|
}`
|
|
5431
5686
|
);
|
|
5432
5687
|
}
|
|
5433
|
-
if (updateOk &&
|
|
5688
|
+
if (updateOk && config.verbose) {
|
|
5434
5689
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5435
5690
|
() => targetResource.get(originalId, { skipCache: true })
|
|
5436
5691
|
);
|
|
@@ -5471,7 +5726,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5471
5726
|
}
|
|
5472
5727
|
if (updateOk) {
|
|
5473
5728
|
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5474
|
-
const
|
|
5729
|
+
const markAppliedConcurrency = config.markAppliedConcurrency || 50;
|
|
5730
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
|
|
5475
5731
|
const [ok2, err2] = await tryFn(
|
|
5476
5732
|
() => transactionResource.update(txn.id, { applied: true })
|
|
5477
5733
|
);
|
|
@@ -5519,9 +5775,11 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5519
5775
|
}
|
|
5520
5776
|
return consolidatedValue;
|
|
5521
5777
|
} finally {
|
|
5522
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5778
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5779
|
+
() => storage.releaseLock(lockKey)
|
|
5780
|
+
);
|
|
5523
5781
|
if (!lockReleased && config.verbose) {
|
|
5524
|
-
console.warn(`[EventualConsistency] Failed to release lock ${
|
|
5782
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5525
5783
|
}
|
|
5526
5784
|
}
|
|
5527
5785
|
}
|
|
@@ -5595,17 +5853,15 @@ async function getCohortStats(cohortDate, transactionResource) {
|
|
|
5595
5853
|
}
|
|
5596
5854
|
return stats;
|
|
5597
5855
|
}
|
|
5598
|
-
async function recalculateRecord(originalId, transactionResource, targetResource,
|
|
5599
|
-
|
|
5600
|
-
const
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
);
|
|
5608
|
-
if (!lockAcquired) {
|
|
5856
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, storage, consolidateRecordFn, config) {
|
|
5857
|
+
const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
|
|
5858
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5859
|
+
ttl: config.lockTimeout || 30,
|
|
5860
|
+
timeout: 0,
|
|
5861
|
+
// Don't wait if locked
|
|
5862
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5863
|
+
});
|
|
5864
|
+
if (!lock) {
|
|
5609
5865
|
if (config.verbose) {
|
|
5610
5866
|
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5611
5867
|
}
|
|
@@ -5673,9 +5929,11 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5673
5929
|
}
|
|
5674
5930
|
return consolidatedValue;
|
|
5675
5931
|
} finally {
|
|
5676
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5932
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5933
|
+
() => storage.releaseLock(lockKey)
|
|
5934
|
+
);
|
|
5677
5935
|
if (!lockReleased && config.verbose) {
|
|
5678
|
-
console.warn(`[EventualConsistency] Failed to release recalculate lock ${
|
|
5936
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5679
5937
|
}
|
|
5680
5938
|
}
|
|
5681
5939
|
}
|
|
@@ -5687,16 +5945,16 @@ function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCall
|
|
|
5687
5945
|
}, gcIntervalMs);
|
|
5688
5946
|
return handler.gcTimer;
|
|
5689
5947
|
}
|
|
5690
|
-
async function runGarbageCollection(transactionResource,
|
|
5691
|
-
const
|
|
5692
|
-
const
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
);
|
|
5699
|
-
if (!
|
|
5948
|
+
async function runGarbageCollection(transactionResource, storage, config, emitFn) {
|
|
5949
|
+
const lockKey = `gc-${config.resource}-${config.field}`;
|
|
5950
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5951
|
+
ttl: 300,
|
|
5952
|
+
// 5 minutes for GC
|
|
5953
|
+
timeout: 0,
|
|
5954
|
+
// Don't wait if locked
|
|
5955
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5956
|
+
});
|
|
5957
|
+
if (!lock) {
|
|
5700
5958
|
if (config.verbose) {
|
|
5701
5959
|
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5702
5960
|
}
|
|
@@ -5754,7 +6012,7 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
|
|
|
5754
6012
|
emitFn("eventual-consistency.gc-error", error);
|
|
5755
6013
|
}
|
|
5756
6014
|
} finally {
|
|
5757
|
-
await tryFn(() =>
|
|
6015
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
5758
6016
|
}
|
|
5759
6017
|
}
|
|
5760
6018
|
|
|
@@ -5764,12 +6022,12 @@ async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
5764
6022
|
throw new Error(
|
|
5765
6023
|
`[EventualConsistency] CRITICAL BUG: config.field is undefined in updateAnalytics()!
|
|
5766
6024
|
This indicates a race condition in the plugin where multiple handlers are sharing the same config object.
|
|
5767
|
-
Config: ${JSON.stringify({ resource: config.resource, field: config.field
|
|
6025
|
+
Config: ${JSON.stringify({ resource: config.resource, field: config.field })}
|
|
5768
6026
|
Transactions count: ${transactions.length}
|
|
5769
6027
|
AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
5770
6028
|
);
|
|
5771
6029
|
}
|
|
5772
|
-
if (config.verbose
|
|
6030
|
+
if (config.verbose) {
|
|
5773
6031
|
console.log(
|
|
5774
6032
|
`[EventualConsistency] ${config.resource}.${config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5775
6033
|
);
|
|
@@ -5777,26 +6035,30 @@ AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
|
5777
6035
|
try {
|
|
5778
6036
|
const byHour = groupByCohort(transactions, "cohortHour");
|
|
5779
6037
|
const cohortCount = Object.keys(byHour).length;
|
|
5780
|
-
if (config.verbose
|
|
6038
|
+
if (config.verbose) {
|
|
5781
6039
|
console.log(
|
|
5782
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
6040
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
|
|
5783
6041
|
);
|
|
5784
6042
|
}
|
|
5785
|
-
|
|
5786
|
-
|
|
5787
|
-
|
|
6043
|
+
await Promise.all(
|
|
6044
|
+
Object.entries(byHour).map(
|
|
6045
|
+
([cohort, txns]) => upsertAnalytics("hour", cohort, txns, analyticsResource, config)
|
|
6046
|
+
)
|
|
6047
|
+
);
|
|
5788
6048
|
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5789
6049
|
const uniqueHours = Object.keys(byHour);
|
|
5790
|
-
if (config.verbose
|
|
6050
|
+
if (config.verbose) {
|
|
5791
6051
|
console.log(
|
|
5792
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
6052
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
|
|
5793
6053
|
);
|
|
5794
6054
|
}
|
|
5795
|
-
|
|
5796
|
-
|
|
5797
|
-
|
|
6055
|
+
await Promise.all(
|
|
6056
|
+
uniqueHours.map(
|
|
6057
|
+
(cohortHour) => rollupAnalytics(cohortHour, analyticsResource, config)
|
|
6058
|
+
)
|
|
6059
|
+
);
|
|
5798
6060
|
}
|
|
5799
|
-
if (config.verbose
|
|
6061
|
+
if (config.verbose) {
|
|
5800
6062
|
console.log(
|
|
5801
6063
|
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5802
6064
|
);
|
|
@@ -5897,18 +6159,53 @@ function calculateOperationBreakdown(transactions) {
|
|
|
5897
6159
|
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5898
6160
|
const cohortDate = cohortHour.substring(0, 10);
|
|
5899
6161
|
const cohortMonth = cohortHour.substring(0, 7);
|
|
6162
|
+
const date = new Date(cohortDate);
|
|
6163
|
+
const cohortWeek = getCohortWeekFromDate(date);
|
|
5900
6164
|
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
|
|
6165
|
+
await rollupPeriod("week", cohortWeek, cohortWeek, analyticsResource, config);
|
|
5901
6166
|
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
|
|
5902
6167
|
}
|
|
6168
|
+
function getCohortWeekFromDate(date) {
|
|
6169
|
+
const target = new Date(date.valueOf());
|
|
6170
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
6171
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
6172
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
6173
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
6174
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
6175
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
6176
|
+
}
|
|
6177
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
6178
|
+
const weekYear = target.getUTCFullYear();
|
|
6179
|
+
return `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
6180
|
+
}
|
|
5903
6181
|
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5904
|
-
|
|
6182
|
+
let sourcePeriod;
|
|
6183
|
+
if (period === "day") {
|
|
6184
|
+
sourcePeriod = "hour";
|
|
6185
|
+
} else if (period === "week") {
|
|
6186
|
+
sourcePeriod = "day";
|
|
6187
|
+
} else if (period === "month") {
|
|
6188
|
+
sourcePeriod = "week";
|
|
6189
|
+
} else {
|
|
6190
|
+
sourcePeriod = "day";
|
|
6191
|
+
}
|
|
5905
6192
|
const [ok, err, allAnalytics] = await tryFn(
|
|
5906
6193
|
() => analyticsResource.list()
|
|
5907
6194
|
);
|
|
5908
6195
|
if (!ok || !allAnalytics) return;
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
6196
|
+
let sourceAnalytics;
|
|
6197
|
+
if (period === "week") {
|
|
6198
|
+
sourceAnalytics = allAnalytics.filter((a) => {
|
|
6199
|
+
if (a.period !== sourcePeriod) return false;
|
|
6200
|
+
const dayDate = new Date(a.cohort);
|
|
6201
|
+
const dayWeek = getCohortWeekFromDate(dayDate);
|
|
6202
|
+
return dayWeek === cohort;
|
|
6203
|
+
});
|
|
6204
|
+
} else {
|
|
6205
|
+
sourceAnalytics = allAnalytics.filter(
|
|
6206
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
6207
|
+
);
|
|
6208
|
+
}
|
|
5912
6209
|
if (sourceAnalytics.length === 0) return;
|
|
5913
6210
|
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5914
6211
|
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
@@ -6117,6 +6414,32 @@ async function getYearByMonth(resourceName, field, year, options, fieldHandlers)
|
|
|
6117
6414
|
}
|
|
6118
6415
|
return data;
|
|
6119
6416
|
}
|
|
6417
|
+
async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
|
|
6418
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6419
|
+
period: "week",
|
|
6420
|
+
year
|
|
6421
|
+
}, fieldHandlers);
|
|
6422
|
+
if (options.fillGaps) {
|
|
6423
|
+
const startWeek = `${year}-W01`;
|
|
6424
|
+
const endWeek = `${year}-W53`;
|
|
6425
|
+
return fillGaps(data, "week", startWeek, endWeek);
|
|
6426
|
+
}
|
|
6427
|
+
return data;
|
|
6428
|
+
}
|
|
6429
|
+
async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
|
|
6430
|
+
const year = parseInt(month.substring(0, 4));
|
|
6431
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
6432
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
6433
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
6434
|
+
const firstWeek = getCohortWeekFromDate(firstDay);
|
|
6435
|
+
const lastWeek = getCohortWeekFromDate(lastDay);
|
|
6436
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6437
|
+
period: "week",
|
|
6438
|
+
startDate: firstWeek,
|
|
6439
|
+
endDate: lastWeek
|
|
6440
|
+
}, fieldHandlers);
|
|
6441
|
+
return data;
|
|
6442
|
+
}
|
|
6120
6443
|
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
6121
6444
|
let year, monthNum;
|
|
6122
6445
|
if (month === "last") {
|
|
@@ -6344,7 +6667,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6344
6667
|
if (!handler.targetResource) return;
|
|
6345
6668
|
const resourceName = handler.resource;
|
|
6346
6669
|
const fieldName = handler.field;
|
|
6347
|
-
const transactionResourceName =
|
|
6670
|
+
const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
|
|
6348
6671
|
const partitionConfig = createPartitionConfig();
|
|
6349
6672
|
const [ok, err, transactionResource] = await tryFn(
|
|
6350
6673
|
() => database.createResource({
|
|
@@ -6358,6 +6681,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6358
6681
|
timestamp: "string|required",
|
|
6359
6682
|
cohortDate: "string|required",
|
|
6360
6683
|
cohortHour: "string|required",
|
|
6684
|
+
cohortWeek: "string|optional",
|
|
6361
6685
|
cohortMonth: "string|optional",
|
|
6362
6686
|
source: "string|optional",
|
|
6363
6687
|
applied: "boolean|optional"
|
|
@@ -6373,36 +6697,18 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6373
6697
|
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
6374
6698
|
}
|
|
6375
6699
|
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
6376
|
-
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
6377
|
-
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
6378
|
-
() => database.createResource({
|
|
6379
|
-
name: lockResourceName,
|
|
6380
|
-
attributes: {
|
|
6381
|
-
id: "string|required",
|
|
6382
|
-
lockedAt: "number|required",
|
|
6383
|
-
workerId: "string|optional"
|
|
6384
|
-
},
|
|
6385
|
-
behavior: "body-only",
|
|
6386
|
-
timestamps: false,
|
|
6387
|
-
createdBy: "EventualConsistencyPlugin"
|
|
6388
|
-
})
|
|
6389
|
-
);
|
|
6390
|
-
if (!lockOk && !database.resources[lockResourceName]) {
|
|
6391
|
-
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
6392
|
-
}
|
|
6393
|
-
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
6394
6700
|
if (config.enableAnalytics) {
|
|
6395
6701
|
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
6396
6702
|
}
|
|
6397
6703
|
addHelperMethodsForHandler(handler, plugin, config);
|
|
6398
6704
|
if (config.verbose) {
|
|
6399
6705
|
console.log(
|
|
6400
|
-
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}
|
|
6706
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ""} (locks via PluginStorage TTL)`
|
|
6401
6707
|
);
|
|
6402
6708
|
}
|
|
6403
6709
|
}
|
|
6404
6710
|
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
6405
|
-
const analyticsResourceName =
|
|
6711
|
+
const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
|
|
6406
6712
|
const [ok, err, analyticsResource] = await tryFn(
|
|
6407
6713
|
() => database.createResource({
|
|
6408
6714
|
name: analyticsResourceName,
|
|
@@ -6576,7 +6882,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6576
6882
|
originalId,
|
|
6577
6883
|
this.transactionResource,
|
|
6578
6884
|
this.targetResource,
|
|
6579
|
-
this.
|
|
6885
|
+
this.getStorage(),
|
|
6580
6886
|
this.analyticsResource,
|
|
6581
6887
|
(transactions) => this.updateAnalytics(transactions),
|
|
6582
6888
|
this.config
|
|
@@ -6612,7 +6918,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6612
6918
|
originalId,
|
|
6613
6919
|
this.transactionResource,
|
|
6614
6920
|
this.targetResource,
|
|
6615
|
-
this.
|
|
6921
|
+
this.getStorage(),
|
|
6616
6922
|
(id) => this.consolidateRecord(id),
|
|
6617
6923
|
this.config
|
|
6618
6924
|
);
|
|
@@ -6633,20 +6939,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6633
6939
|
const oldField = this.config.field;
|
|
6634
6940
|
const oldTransactionResource = this.transactionResource;
|
|
6635
6941
|
const oldTargetResource = this.targetResource;
|
|
6636
|
-
const oldLockResource = this.lockResource;
|
|
6637
6942
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6638
6943
|
this.config.resource = handler.resource;
|
|
6639
6944
|
this.config.field = handler.field;
|
|
6640
6945
|
this.transactionResource = handler.transactionResource;
|
|
6641
6946
|
this.targetResource = handler.targetResource;
|
|
6642
|
-
this.lockResource = handler.lockResource;
|
|
6643
6947
|
this.analyticsResource = handler.analyticsResource;
|
|
6644
6948
|
const result = await this.consolidateRecord(id);
|
|
6645
6949
|
this.config.resource = oldResource;
|
|
6646
6950
|
this.config.field = oldField;
|
|
6647
6951
|
this.transactionResource = oldTransactionResource;
|
|
6648
6952
|
this.targetResource = oldTargetResource;
|
|
6649
|
-
this.lockResource = oldLockResource;
|
|
6650
6953
|
this.analyticsResource = oldAnalyticsResource;
|
|
6651
6954
|
return result;
|
|
6652
6955
|
}
|
|
@@ -6659,20 +6962,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6659
6962
|
const oldField = this.config.field;
|
|
6660
6963
|
const oldTransactionResource = this.transactionResource;
|
|
6661
6964
|
const oldTargetResource = this.targetResource;
|
|
6662
|
-
const oldLockResource = this.lockResource;
|
|
6663
6965
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6664
6966
|
this.config.resource = handler.resource;
|
|
6665
6967
|
this.config.field = handler.field;
|
|
6666
6968
|
this.transactionResource = handler.transactionResource;
|
|
6667
6969
|
this.targetResource = handler.targetResource;
|
|
6668
|
-
this.lockResource = handler.lockResource;
|
|
6669
6970
|
this.analyticsResource = handler.analyticsResource;
|
|
6670
6971
|
const result = await this.consolidateRecord(id);
|
|
6671
6972
|
this.config.resource = oldResource;
|
|
6672
6973
|
this.config.field = oldField;
|
|
6673
6974
|
this.transactionResource = oldTransactionResource;
|
|
6674
6975
|
this.targetResource = oldTargetResource;
|
|
6675
|
-
this.lockResource = oldLockResource;
|
|
6676
6976
|
this.analyticsResource = oldAnalyticsResource;
|
|
6677
6977
|
return result;
|
|
6678
6978
|
}
|
|
@@ -6705,20 +7005,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6705
7005
|
const oldField = this.config.field;
|
|
6706
7006
|
const oldTransactionResource = this.transactionResource;
|
|
6707
7007
|
const oldTargetResource = this.targetResource;
|
|
6708
|
-
const oldLockResource = this.lockResource;
|
|
6709
7008
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6710
7009
|
this.config.resource = handler.resource;
|
|
6711
7010
|
this.config.field = handler.field;
|
|
6712
7011
|
this.transactionResource = handler.transactionResource;
|
|
6713
7012
|
this.targetResource = handler.targetResource;
|
|
6714
|
-
this.lockResource = handler.lockResource;
|
|
6715
7013
|
this.analyticsResource = handler.analyticsResource;
|
|
6716
7014
|
const result = await this.recalculateRecord(id);
|
|
6717
7015
|
this.config.resource = oldResource;
|
|
6718
7016
|
this.config.field = oldField;
|
|
6719
7017
|
this.transactionResource = oldTransactionResource;
|
|
6720
7018
|
this.targetResource = oldTargetResource;
|
|
6721
|
-
this.lockResource = oldLockResource;
|
|
6722
7019
|
this.analyticsResource = oldAnalyticsResource;
|
|
6723
7020
|
return result;
|
|
6724
7021
|
}
|
|
@@ -6731,13 +7028,11 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6731
7028
|
const oldField = this.config.field;
|
|
6732
7029
|
const oldTransactionResource = this.transactionResource;
|
|
6733
7030
|
const oldTargetResource = this.targetResource;
|
|
6734
|
-
const oldLockResource = this.lockResource;
|
|
6735
7031
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6736
7032
|
this.config.resource = resourceName;
|
|
6737
7033
|
this.config.field = fieldName;
|
|
6738
7034
|
this.transactionResource = handler.transactionResource;
|
|
6739
7035
|
this.targetResource = handler.targetResource;
|
|
6740
|
-
this.lockResource = handler.lockResource;
|
|
6741
7036
|
this.analyticsResource = handler.analyticsResource;
|
|
6742
7037
|
try {
|
|
6743
7038
|
await runConsolidation(
|
|
@@ -6751,7 +7046,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6751
7046
|
this.config.field = oldField;
|
|
6752
7047
|
this.transactionResource = oldTransactionResource;
|
|
6753
7048
|
this.targetResource = oldTargetResource;
|
|
6754
|
-
this.lockResource = oldLockResource;
|
|
6755
7049
|
this.analyticsResource = oldAnalyticsResource;
|
|
6756
7050
|
}
|
|
6757
7051
|
}
|
|
@@ -6764,16 +7058,14 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6764
7058
|
const oldField = this.config.field;
|
|
6765
7059
|
const oldTransactionResource = this.transactionResource;
|
|
6766
7060
|
const oldTargetResource = this.targetResource;
|
|
6767
|
-
const oldLockResource = this.lockResource;
|
|
6768
7061
|
this.config.resource = resourceName;
|
|
6769
7062
|
this.config.field = fieldName;
|
|
6770
7063
|
this.transactionResource = handler.transactionResource;
|
|
6771
7064
|
this.targetResource = handler.targetResource;
|
|
6772
|
-
this.lockResource = handler.lockResource;
|
|
6773
7065
|
try {
|
|
6774
7066
|
await runGarbageCollection(
|
|
6775
7067
|
this.transactionResource,
|
|
6776
|
-
this.
|
|
7068
|
+
this.getStorage(),
|
|
6777
7069
|
this.config,
|
|
6778
7070
|
(event, data) => this.emit(event, data)
|
|
6779
7071
|
);
|
|
@@ -6782,7 +7074,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6782
7074
|
this.config.field = oldField;
|
|
6783
7075
|
this.transactionResource = oldTransactionResource;
|
|
6784
7076
|
this.targetResource = oldTargetResource;
|
|
6785
|
-
this.lockResource = oldLockResource;
|
|
6786
7077
|
}
|
|
6787
7078
|
}
|
|
6788
7079
|
// Public Analytics API
|
|
@@ -6851,6 +7142,28 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6851
7142
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
6852
7143
|
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6853
7144
|
}
|
|
7145
|
+
/**
|
|
7146
|
+
* Get analytics for entire year, broken down by weeks
|
|
7147
|
+
* @param {string} resourceName - Resource name
|
|
7148
|
+
* @param {string} field - Field name
|
|
7149
|
+
* @param {number} year - Year (e.g., 2025)
|
|
7150
|
+
* @param {Object} options - Options
|
|
7151
|
+
* @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
|
|
7152
|
+
*/
|
|
7153
|
+
async getYearByWeek(resourceName, field, year, options = {}) {
|
|
7154
|
+
return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
|
|
7155
|
+
}
|
|
7156
|
+
/**
|
|
7157
|
+
* Get analytics for entire month, broken down by weeks
|
|
7158
|
+
* @param {string} resourceName - Resource name
|
|
7159
|
+
* @param {string} field - Field name
|
|
7160
|
+
* @param {string} month - Month in YYYY-MM format
|
|
7161
|
+
* @param {Object} options - Options
|
|
7162
|
+
* @returns {Promise<Array>} Weekly analytics for the month
|
|
7163
|
+
*/
|
|
7164
|
+
async getMonthByWeek(resourceName, field, month, options = {}) {
|
|
7165
|
+
return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
|
|
7166
|
+
}
|
|
6854
7167
|
/**
|
|
6855
7168
|
* Get top records by volume
|
|
6856
7169
|
* @param {string} resourceName - Resource name
|
|
@@ -6873,6 +7186,8 @@ class FullTextPlugin extends Plugin {
|
|
|
6873
7186
|
...options
|
|
6874
7187
|
};
|
|
6875
7188
|
this.indexes = /* @__PURE__ */ new Map();
|
|
7189
|
+
this.dirtyIndexes = /* @__PURE__ */ new Set();
|
|
7190
|
+
this.deletedIndexes = /* @__PURE__ */ new Set();
|
|
6876
7191
|
}
|
|
6877
7192
|
async onInstall() {
|
|
6878
7193
|
const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
|
|
@@ -6886,7 +7201,11 @@ class FullTextPlugin extends Plugin {
|
|
|
6886
7201
|
// Array of record IDs containing this word
|
|
6887
7202
|
count: "number|required",
|
|
6888
7203
|
lastUpdated: "string|required"
|
|
6889
|
-
}
|
|
7204
|
+
},
|
|
7205
|
+
partitions: {
|
|
7206
|
+
byResource: { fields: { resourceName: "string" } }
|
|
7207
|
+
},
|
|
7208
|
+
behavior: "body-overflow"
|
|
6890
7209
|
}));
|
|
6891
7210
|
this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
|
|
6892
7211
|
await this.loadIndexes();
|
|
@@ -6915,22 +7234,53 @@ class FullTextPlugin extends Plugin {
|
|
|
6915
7234
|
async saveIndexes() {
|
|
6916
7235
|
if (!this.indexResource) return;
|
|
6917
7236
|
const [ok, err] = await tryFn(async () => {
|
|
6918
|
-
const
|
|
6919
|
-
|
|
6920
|
-
await
|
|
7237
|
+
for (const key of this.deletedIndexes) {
|
|
7238
|
+
const [resourceName] = key.split(":");
|
|
7239
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7240
|
+
() => this.indexResource.query({ resourceName })
|
|
7241
|
+
);
|
|
7242
|
+
if (queryOk && results) {
|
|
7243
|
+
for (const index of results) {
|
|
7244
|
+
const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
|
|
7245
|
+
if (indexKey === key) {
|
|
7246
|
+
await this.indexResource.delete(index.id);
|
|
7247
|
+
}
|
|
7248
|
+
}
|
|
7249
|
+
}
|
|
6921
7250
|
}
|
|
6922
|
-
for (const
|
|
7251
|
+
for (const key of this.dirtyIndexes) {
|
|
6923
7252
|
const [resourceName, fieldName, word] = key.split(":");
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
7253
|
+
const data = this.indexes.get(key);
|
|
7254
|
+
if (!data) continue;
|
|
7255
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7256
|
+
() => this.indexResource.query({ resourceName })
|
|
7257
|
+
);
|
|
7258
|
+
let existingRecord = null;
|
|
7259
|
+
if (queryOk && results) {
|
|
7260
|
+
existingRecord = results.find(
|
|
7261
|
+
(index) => index.resourceName === resourceName && index.fieldName === fieldName && index.word === word
|
|
7262
|
+
);
|
|
7263
|
+
}
|
|
7264
|
+
if (existingRecord) {
|
|
7265
|
+
await this.indexResource.update(existingRecord.id, {
|
|
7266
|
+
recordIds: data.recordIds,
|
|
7267
|
+
count: data.count,
|
|
7268
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7269
|
+
});
|
|
7270
|
+
} else {
|
|
7271
|
+
await this.indexResource.insert({
|
|
7272
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
7273
|
+
resourceName,
|
|
7274
|
+
fieldName,
|
|
7275
|
+
word,
|
|
7276
|
+
recordIds: data.recordIds,
|
|
7277
|
+
count: data.count,
|
|
7278
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7279
|
+
});
|
|
7280
|
+
}
|
|
6933
7281
|
}
|
|
7282
|
+
this.dirtyIndexes.clear();
|
|
7283
|
+
this.deletedIndexes.clear();
|
|
6934
7284
|
});
|
|
6935
7285
|
}
|
|
6936
7286
|
installDatabaseHooks() {
|
|
@@ -7025,6 +7375,7 @@ class FullTextPlugin extends Plugin {
|
|
|
7025
7375
|
existing.count = existing.recordIds.length;
|
|
7026
7376
|
}
|
|
7027
7377
|
this.indexes.set(key, existing);
|
|
7378
|
+
this.dirtyIndexes.add(key);
|
|
7028
7379
|
}
|
|
7029
7380
|
}
|
|
7030
7381
|
}
|
|
@@ -7037,8 +7388,10 @@ class FullTextPlugin extends Plugin {
|
|
|
7037
7388
|
data.count = data.recordIds.length;
|
|
7038
7389
|
if (data.recordIds.length === 0) {
|
|
7039
7390
|
this.indexes.delete(key);
|
|
7391
|
+
this.deletedIndexes.add(key);
|
|
7040
7392
|
} else {
|
|
7041
7393
|
this.indexes.set(key, data);
|
|
7394
|
+
this.dirtyIndexes.add(key);
|
|
7042
7395
|
}
|
|
7043
7396
|
}
|
|
7044
7397
|
}
|
|
@@ -7270,8 +7623,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7270
7623
|
errors: "number|required",
|
|
7271
7624
|
avgTime: "number|required",
|
|
7272
7625
|
timestamp: "string|required",
|
|
7273
|
-
metadata: "json"
|
|
7274
|
-
|
|
7626
|
+
metadata: "json",
|
|
7627
|
+
createdAt: "string|required"
|
|
7628
|
+
// YYYY-MM-DD for partitioning
|
|
7629
|
+
},
|
|
7630
|
+
partitions: {
|
|
7631
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7632
|
+
},
|
|
7633
|
+
behavior: "body-overflow"
|
|
7275
7634
|
}));
|
|
7276
7635
|
this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
|
|
7277
7636
|
const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7282,8 +7641,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7282
7641
|
operation: "string|required",
|
|
7283
7642
|
error: "string|required",
|
|
7284
7643
|
timestamp: "string|required",
|
|
7285
|
-
metadata: "json"
|
|
7286
|
-
|
|
7644
|
+
metadata: "json",
|
|
7645
|
+
createdAt: "string|required"
|
|
7646
|
+
// YYYY-MM-DD for partitioning
|
|
7647
|
+
},
|
|
7648
|
+
partitions: {
|
|
7649
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7650
|
+
},
|
|
7651
|
+
behavior: "body-overflow"
|
|
7287
7652
|
}));
|
|
7288
7653
|
this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
|
|
7289
7654
|
const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7294,8 +7659,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7294
7659
|
operation: "string|required",
|
|
7295
7660
|
duration: "number|required",
|
|
7296
7661
|
timestamp: "string|required",
|
|
7297
|
-
metadata: "json"
|
|
7298
|
-
|
|
7662
|
+
metadata: "json",
|
|
7663
|
+
createdAt: "string|required"
|
|
7664
|
+
// YYYY-MM-DD for partitioning
|
|
7665
|
+
},
|
|
7666
|
+
partitions: {
|
|
7667
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7668
|
+
},
|
|
7669
|
+
behavior: "body-overflow"
|
|
7299
7670
|
}));
|
|
7300
7671
|
this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
|
|
7301
7672
|
});
|
|
@@ -7516,6 +7887,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7516
7887
|
errorMetadata = { error: "true" };
|
|
7517
7888
|
resourceMetadata = { resource: "true" };
|
|
7518
7889
|
}
|
|
7890
|
+
const now = /* @__PURE__ */ new Date();
|
|
7891
|
+
const createdAt = now.toISOString().slice(0, 10);
|
|
7519
7892
|
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
7520
7893
|
if (data.count > 0) {
|
|
7521
7894
|
await this.metricsResource.insert({
|
|
@@ -7527,7 +7900,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7527
7900
|
totalTime: data.totalTime,
|
|
7528
7901
|
errors: data.errors,
|
|
7529
7902
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7530
|
-
timestamp:
|
|
7903
|
+
timestamp: now.toISOString(),
|
|
7904
|
+
createdAt,
|
|
7531
7905
|
metadata
|
|
7532
7906
|
});
|
|
7533
7907
|
}
|
|
@@ -7544,7 +7918,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7544
7918
|
totalTime: data.totalTime,
|
|
7545
7919
|
errors: data.errors,
|
|
7546
7920
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7547
|
-
timestamp:
|
|
7921
|
+
timestamp: now.toISOString(),
|
|
7922
|
+
createdAt,
|
|
7548
7923
|
metadata: resourceMetadata
|
|
7549
7924
|
});
|
|
7550
7925
|
}
|
|
@@ -7558,6 +7933,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7558
7933
|
operation: perf.operation,
|
|
7559
7934
|
duration: perf.duration,
|
|
7560
7935
|
timestamp: perf.timestamp,
|
|
7936
|
+
createdAt: perf.timestamp.slice(0, 10),
|
|
7937
|
+
// YYYY-MM-DD from timestamp
|
|
7561
7938
|
metadata: perfMetadata
|
|
7562
7939
|
});
|
|
7563
7940
|
}
|
|
@@ -7571,6 +7948,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7571
7948
|
error: error.error,
|
|
7572
7949
|
stack: error.stack,
|
|
7573
7950
|
timestamp: error.timestamp,
|
|
7951
|
+
createdAt: error.timestamp.slice(0, 10),
|
|
7952
|
+
// YYYY-MM-DD from timestamp
|
|
7574
7953
|
metadata: errorMetadata
|
|
7575
7954
|
});
|
|
7576
7955
|
}
|
|
@@ -7702,22 +8081,47 @@ class MetricsPlugin extends Plugin {
|
|
|
7702
8081
|
async cleanupOldData() {
|
|
7703
8082
|
const cutoffDate = /* @__PURE__ */ new Date();
|
|
7704
8083
|
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
8084
|
+
cutoffDate.toISOString().slice(0, 10);
|
|
8085
|
+
const datesToDelete = [];
|
|
8086
|
+
const startDate = new Date(cutoffDate);
|
|
8087
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
8088
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
8089
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
8090
|
+
}
|
|
7705
8091
|
if (this.metricsResource) {
|
|
7706
|
-
const
|
|
7707
|
-
|
|
7708
|
-
|
|
8092
|
+
for (const dateStr of datesToDelete) {
|
|
8093
|
+
const [ok, err, oldMetrics] = await tryFn(
|
|
8094
|
+
() => this.metricsResource.query({ createdAt: dateStr })
|
|
8095
|
+
);
|
|
8096
|
+
if (ok && oldMetrics) {
|
|
8097
|
+
for (const metric of oldMetrics) {
|
|
8098
|
+
await tryFn(() => this.metricsResource.delete(metric.id));
|
|
8099
|
+
}
|
|
8100
|
+
}
|
|
7709
8101
|
}
|
|
7710
8102
|
}
|
|
7711
8103
|
if (this.errorsResource) {
|
|
7712
|
-
const
|
|
7713
|
-
|
|
7714
|
-
|
|
8104
|
+
for (const dateStr of datesToDelete) {
|
|
8105
|
+
const [ok, err, oldErrors] = await tryFn(
|
|
8106
|
+
() => this.errorsResource.query({ createdAt: dateStr })
|
|
8107
|
+
);
|
|
8108
|
+
if (ok && oldErrors) {
|
|
8109
|
+
for (const error of oldErrors) {
|
|
8110
|
+
await tryFn(() => this.errorsResource.delete(error.id));
|
|
8111
|
+
}
|
|
8112
|
+
}
|
|
7715
8113
|
}
|
|
7716
8114
|
}
|
|
7717
8115
|
if (this.performanceResource) {
|
|
7718
|
-
const
|
|
7719
|
-
|
|
7720
|
-
|
|
8116
|
+
for (const dateStr of datesToDelete) {
|
|
8117
|
+
const [ok, err, oldPerformance] = await tryFn(
|
|
8118
|
+
() => this.performanceResource.query({ createdAt: dateStr })
|
|
8119
|
+
);
|
|
8120
|
+
if (ok && oldPerformance) {
|
|
8121
|
+
for (const perf of oldPerformance) {
|
|
8122
|
+
await tryFn(() => this.performanceResource.delete(perf.id));
|
|
8123
|
+
}
|
|
8124
|
+
}
|
|
7721
8125
|
}
|
|
7722
8126
|
}
|
|
7723
8127
|
}
|
|
@@ -12788,7 +13192,7 @@ class Database extends EventEmitter {
|
|
|
12788
13192
|
this.id = idGenerator(7);
|
|
12789
13193
|
this.version = "1";
|
|
12790
13194
|
this.s3dbVersion = (() => {
|
|
12791
|
-
const [ok, err, version] = tryFn(() => true ? "11.0.
|
|
13195
|
+
const [ok, err, version] = tryFn(() => true ? "11.0.3" : "latest");
|
|
12792
13196
|
return ok ? version : "latest";
|
|
12793
13197
|
})();
|
|
12794
13198
|
this.resources = {};
|
|
@@ -15097,28 +15501,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15097
15501
|
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
15098
15502
|
}
|
|
15099
15503
|
this.queueResource = this.database.resources[queueName];
|
|
15100
|
-
const lockName = `${this.config.resource}_locks`;
|
|
15101
|
-
const [okLock, errLock] = await tryFn(
|
|
15102
|
-
() => this.database.createResource({
|
|
15103
|
-
name: lockName,
|
|
15104
|
-
attributes: {
|
|
15105
|
-
id: "string|required",
|
|
15106
|
-
workerId: "string|required",
|
|
15107
|
-
timestamp: "number|required",
|
|
15108
|
-
ttl: "number|default:5000"
|
|
15109
|
-
},
|
|
15110
|
-
behavior: "body-overflow",
|
|
15111
|
-
timestamps: false
|
|
15112
|
-
})
|
|
15113
|
-
);
|
|
15114
|
-
if (okLock || this.database.resources[lockName]) {
|
|
15115
|
-
this.lockResource = this.database.resources[lockName];
|
|
15116
|
-
} else {
|
|
15117
|
-
this.lockResource = null;
|
|
15118
|
-
if (this.config.verbose) {
|
|
15119
|
-
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
15120
|
-
}
|
|
15121
|
-
}
|
|
15122
15504
|
this.addHelperMethods();
|
|
15123
15505
|
if (this.config.deadLetterResource) {
|
|
15124
15506
|
await this.createDeadLetterResource();
|
|
@@ -15189,13 +15571,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15189
15571
|
}
|
|
15190
15572
|
}
|
|
15191
15573
|
}, 5e3);
|
|
15192
|
-
this.lockCleanupInterval = setInterval(() => {
|
|
15193
|
-
this.cleanupStaleLocks().catch((err) => {
|
|
15194
|
-
if (this.config.verbose) {
|
|
15195
|
-
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
15196
|
-
}
|
|
15197
|
-
});
|
|
15198
|
-
}, 1e4);
|
|
15199
15574
|
for (let i = 0; i < concurrency; i++) {
|
|
15200
15575
|
const worker = this.createWorker(messageHandler, i);
|
|
15201
15576
|
this.workers.push(worker);
|
|
@@ -15212,10 +15587,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15212
15587
|
clearInterval(this.cacheCleanupInterval);
|
|
15213
15588
|
this.cacheCleanupInterval = null;
|
|
15214
15589
|
}
|
|
15215
|
-
if (this.lockCleanupInterval) {
|
|
15216
|
-
clearInterval(this.lockCleanupInterval);
|
|
15217
|
-
this.lockCleanupInterval = null;
|
|
15218
|
-
}
|
|
15219
15590
|
await Promise.all(this.workers);
|
|
15220
15591
|
this.workers = [];
|
|
15221
15592
|
this.processedCache.clear();
|
|
@@ -15266,48 +15637,21 @@ class S3QueuePlugin extends Plugin {
|
|
|
15266
15637
|
return null;
|
|
15267
15638
|
}
|
|
15268
15639
|
/**
|
|
15269
|
-
* Acquire a distributed lock using
|
|
15640
|
+
* Acquire a distributed lock using PluginStorage TTL
|
|
15270
15641
|
* This ensures only one worker can claim a message at a time
|
|
15271
|
-
*
|
|
15272
|
-
* Uses a two-step process:
|
|
15273
|
-
* 1. Create lock resource (similar to queue resource) if not exists
|
|
15274
|
-
* 2. Try to claim lock using ETag-based conditional update
|
|
15275
15642
|
*/
|
|
15276
15643
|
async acquireLock(messageId) {
|
|
15277
|
-
|
|
15278
|
-
|
|
15279
|
-
}
|
|
15280
|
-
const lockId = `lock-${messageId}`;
|
|
15281
|
-
const now = Date.now();
|
|
15644
|
+
const storage = this.getStorage();
|
|
15645
|
+
const lockKey = `msg-${messageId}`;
|
|
15282
15646
|
try {
|
|
15283
|
-
const
|
|
15284
|
-
|
|
15285
|
-
|
|
15286
|
-
|
|
15287
|
-
|
|
15288
|
-
|
|
15289
|
-
|
|
15290
|
-
|
|
15291
|
-
const [ok, err, result] = await tryFn(
|
|
15292
|
-
() => this.lockResource.updateConditional(lockId, {
|
|
15293
|
-
workerId: this.workerId,
|
|
15294
|
-
timestamp: now,
|
|
15295
|
-
ttl: 5e3
|
|
15296
|
-
}, {
|
|
15297
|
-
ifMatch: existingLock._etag
|
|
15298
|
-
})
|
|
15299
|
-
);
|
|
15300
|
-
return ok && result.success;
|
|
15301
|
-
}
|
|
15302
|
-
const [okCreate, errCreate] = await tryFn(
|
|
15303
|
-
() => this.lockResource.insert({
|
|
15304
|
-
id: lockId,
|
|
15305
|
-
workerId: this.workerId,
|
|
15306
|
-
timestamp: now,
|
|
15307
|
-
ttl: 5e3
|
|
15308
|
-
})
|
|
15309
|
-
);
|
|
15310
|
-
return okCreate;
|
|
15647
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
15648
|
+
ttl: 5,
|
|
15649
|
+
// 5 seconds
|
|
15650
|
+
timeout: 0,
|
|
15651
|
+
// Don't wait if locked
|
|
15652
|
+
workerId: this.workerId
|
|
15653
|
+
});
|
|
15654
|
+
return lock !== null;
|
|
15311
15655
|
} catch (error) {
|
|
15312
15656
|
if (this.config.verbose) {
|
|
15313
15657
|
console.log(`[acquireLock] Error: ${error.message}`);
|
|
@@ -15316,15 +15660,13 @@ class S3QueuePlugin extends Plugin {
|
|
|
15316
15660
|
}
|
|
15317
15661
|
}
|
|
15318
15662
|
/**
|
|
15319
|
-
* Release a distributed lock
|
|
15663
|
+
* Release a distributed lock via PluginStorage
|
|
15320
15664
|
*/
|
|
15321
15665
|
async releaseLock(messageId) {
|
|
15322
|
-
|
|
15323
|
-
|
|
15324
|
-
}
|
|
15325
|
-
const lockId = `lock-${messageId}`;
|
|
15666
|
+
const storage = this.getStorage();
|
|
15667
|
+
const lockKey = `msg-${messageId}`;
|
|
15326
15668
|
try {
|
|
15327
|
-
await
|
|
15669
|
+
await storage.releaseLock(lockKey);
|
|
15328
15670
|
} catch (error) {
|
|
15329
15671
|
if (this.config.verbose) {
|
|
15330
15672
|
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
@@ -15332,30 +15674,11 @@ class S3QueuePlugin extends Plugin {
|
|
|
15332
15674
|
}
|
|
15333
15675
|
}
|
|
15334
15676
|
/**
|
|
15335
|
-
* Clean up stale locks
|
|
15336
|
-
*
|
|
15677
|
+
* Clean up stale locks - NO LONGER NEEDED
|
|
15678
|
+
* TTL handles automatic expiration, no manual cleanup required
|
|
15337
15679
|
*/
|
|
15338
15680
|
async cleanupStaleLocks() {
|
|
15339
|
-
|
|
15340
|
-
return;
|
|
15341
|
-
}
|
|
15342
|
-
const now = Date.now();
|
|
15343
|
-
try {
|
|
15344
|
-
const locks = await this.lockResource.list();
|
|
15345
|
-
for (const lock of locks) {
|
|
15346
|
-
const lockAge = now - lock.timestamp;
|
|
15347
|
-
if (lockAge > lock.ttl) {
|
|
15348
|
-
await this.lockResource.delete(lock.id);
|
|
15349
|
-
if (this.config.verbose) {
|
|
15350
|
-
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
15351
|
-
}
|
|
15352
|
-
}
|
|
15353
|
-
}
|
|
15354
|
-
} catch (error) {
|
|
15355
|
-
if (this.config.verbose) {
|
|
15356
|
-
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
15357
|
-
}
|
|
15358
|
-
}
|
|
15681
|
+
return;
|
|
15359
15682
|
}
|
|
15360
15683
|
async attemptClaim(msg) {
|
|
15361
15684
|
const now = Date.now();
|
|
@@ -15579,7 +15902,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15579
15902
|
...options
|
|
15580
15903
|
};
|
|
15581
15904
|
this.database = null;
|
|
15582
|
-
this.lockResource = null;
|
|
15583
15905
|
this.jobs = /* @__PURE__ */ new Map();
|
|
15584
15906
|
this.activeJobs = /* @__PURE__ */ new Map();
|
|
15585
15907
|
this.timers = /* @__PURE__ */ new Map();
|
|
@@ -15618,7 +15940,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15618
15940
|
return true;
|
|
15619
15941
|
}
|
|
15620
15942
|
async onInstall() {
|
|
15621
|
-
await this._createLockResource();
|
|
15622
15943
|
if (this.config.persistJobs) {
|
|
15623
15944
|
await this._createJobHistoryResource();
|
|
15624
15945
|
}
|
|
@@ -15647,25 +15968,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15647
15968
|
await this._startScheduling();
|
|
15648
15969
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
15649
15970
|
}
|
|
15650
|
-
async _createLockResource() {
|
|
15651
|
-
const [ok, err, lockResource] = await tryFn(
|
|
15652
|
-
() => this.database.createResource({
|
|
15653
|
-
name: "plg_scheduler_job_locks",
|
|
15654
|
-
attributes: {
|
|
15655
|
-
id: "string|required",
|
|
15656
|
-
jobName: "string|required",
|
|
15657
|
-
lockedAt: "number|required",
|
|
15658
|
-
instanceId: "string|optional"
|
|
15659
|
-
},
|
|
15660
|
-
behavior: "body-only",
|
|
15661
|
-
timestamps: false
|
|
15662
|
-
})
|
|
15663
|
-
);
|
|
15664
|
-
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
15665
|
-
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
15666
|
-
}
|
|
15667
|
-
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
15668
|
-
}
|
|
15669
15971
|
async _createJobHistoryResource() {
|
|
15670
15972
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
15671
15973
|
name: this.config.jobHistoryResource,
|
|
@@ -15773,16 +16075,16 @@ class SchedulerPlugin extends Plugin {
|
|
|
15773
16075
|
return;
|
|
15774
16076
|
}
|
|
15775
16077
|
this.activeJobs.set(jobName, "acquiring-lock");
|
|
15776
|
-
const
|
|
15777
|
-
const
|
|
15778
|
-
|
|
15779
|
-
|
|
15780
|
-
|
|
15781
|
-
|
|
15782
|
-
|
|
15783
|
-
|
|
15784
|
-
);
|
|
15785
|
-
if (!
|
|
16078
|
+
const storage = this.getStorage();
|
|
16079
|
+
const lockKey = `job-${jobName}`;
|
|
16080
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
16081
|
+
ttl: Math.ceil(job.timeout / 1e3) + 60,
|
|
16082
|
+
// Job timeout + 60 seconds buffer
|
|
16083
|
+
timeout: 0,
|
|
16084
|
+
// Don't wait if locked
|
|
16085
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
16086
|
+
});
|
|
16087
|
+
if (!lock) {
|
|
15786
16088
|
if (this.config.verbose) {
|
|
15787
16089
|
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
15788
16090
|
}
|
|
@@ -15886,7 +16188,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
15886
16188
|
throw lastError;
|
|
15887
16189
|
}
|
|
15888
16190
|
} finally {
|
|
15889
|
-
await tryFn(() =>
|
|
16191
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
15890
16192
|
}
|
|
15891
16193
|
}
|
|
15892
16194
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|