s3db.js 11.0.1 → 11.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +621 -319
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +621 -319
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/concerns/plugin-storage.js +274 -9
- package/src/plugins/audit.plugin.js +94 -18
- package/src/plugins/eventual-consistency/analytics.js +136 -20
- package/src/plugins/eventual-consistency/config.js +4 -2
- package/src/plugins/eventual-consistency/consolidation.js +35 -39
- package/src/plugins/eventual-consistency/garbage-collection.js +11 -13
- package/src/plugins/eventual-consistency/index.js +28 -19
- package/src/plugins/eventual-consistency/install.js +9 -26
- package/src/plugins/eventual-consistency/partitions.js +5 -0
- package/src/plugins/eventual-consistency/transactions.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +36 -1
- package/src/plugins/fulltext.plugin.js +76 -22
- package/src/plugins/metrics.plugin.js +70 -20
- package/src/plugins/s3-queue.plugin.js +21 -120
- package/src/plugins/scheduler.plugin.js +11 -37
- package/dist/s3db-cli.js +0 -54741
package/dist/s3db.es.js
CHANGED
|
@@ -833,18 +833,23 @@ class PluginStorage {
|
|
|
833
833
|
return `plugin=${this.pluginSlug}/${parts.join("/")}`;
|
|
834
834
|
}
|
|
835
835
|
/**
|
|
836
|
-
* Save data with metadata encoding and
|
|
836
|
+
* Save data with metadata encoding, behavior support, and optional TTL
|
|
837
837
|
*
|
|
838
838
|
* @param {string} key - S3 key
|
|
839
839
|
* @param {Object} data - Data to save
|
|
840
840
|
* @param {Object} options - Options
|
|
841
|
+
* @param {number} options.ttl - Time-to-live in seconds (optional)
|
|
841
842
|
* @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
|
|
842
843
|
* @param {string} options.contentType - Content type (default: application/json)
|
|
843
844
|
* @returns {Promise<void>}
|
|
844
845
|
*/
|
|
845
|
-
async
|
|
846
|
-
const { behavior = "body-overflow", contentType = "application/json" } = options;
|
|
847
|
-
const
|
|
846
|
+
async set(key, data, options = {}) {
|
|
847
|
+
const { ttl, behavior = "body-overflow", contentType = "application/json" } = options;
|
|
848
|
+
const dataToSave = { ...data };
|
|
849
|
+
if (ttl && typeof ttl === "number" && ttl > 0) {
|
|
850
|
+
dataToSave._expiresAt = Date.now() + ttl * 1e3;
|
|
851
|
+
}
|
|
852
|
+
const { metadata, body } = this._applyBehavior(dataToSave, behavior);
|
|
848
853
|
const putParams = {
|
|
849
854
|
key,
|
|
850
855
|
metadata,
|
|
@@ -855,14 +860,21 @@ class PluginStorage {
|
|
|
855
860
|
}
|
|
856
861
|
const [ok, err] = await tryFn(() => this.client.putObject(putParams));
|
|
857
862
|
if (!ok) {
|
|
858
|
-
throw new Error(`PluginStorage.
|
|
863
|
+
throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
|
|
859
864
|
}
|
|
860
865
|
}
|
|
861
866
|
/**
|
|
862
|
-
*
|
|
867
|
+
* Alias for set() to maintain backward compatibility
|
|
868
|
+
* @deprecated Use set() instead
|
|
869
|
+
*/
|
|
870
|
+
async put(key, data, options = {}) {
|
|
871
|
+
return this.set(key, data, options);
|
|
872
|
+
}
|
|
873
|
+
/**
|
|
874
|
+
* Get data with automatic metadata decoding and TTL check
|
|
863
875
|
*
|
|
864
876
|
* @param {string} key - S3 key
|
|
865
|
-
* @returns {Promise<Object|null>} Data or null if not found
|
|
877
|
+
* @returns {Promise<Object|null>} Data or null if not found/expired
|
|
866
878
|
*/
|
|
867
879
|
async get(key) {
|
|
868
880
|
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
@@ -874,18 +886,28 @@ class PluginStorage {
|
|
|
874
886
|
}
|
|
875
887
|
const metadata = response.Metadata || {};
|
|
876
888
|
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
889
|
+
let data = parsedMetadata;
|
|
877
890
|
if (response.Body) {
|
|
878
891
|
try {
|
|
879
892
|
const bodyContent = await response.Body.transformToString();
|
|
880
893
|
if (bodyContent && bodyContent.trim()) {
|
|
881
894
|
const body = JSON.parse(bodyContent);
|
|
882
|
-
|
|
895
|
+
data = { ...parsedMetadata, ...body };
|
|
883
896
|
}
|
|
884
897
|
} catch (parseErr) {
|
|
885
898
|
throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
|
|
886
899
|
}
|
|
887
900
|
}
|
|
888
|
-
|
|
901
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
902
|
+
if (expiresAt) {
|
|
903
|
+
if (Date.now() > expiresAt) {
|
|
904
|
+
await this.delete(key);
|
|
905
|
+
return null;
|
|
906
|
+
}
|
|
907
|
+
delete data._expiresat;
|
|
908
|
+
delete data._expiresAt;
|
|
909
|
+
}
|
|
910
|
+
return data;
|
|
889
911
|
}
|
|
890
912
|
/**
|
|
891
913
|
* Parse metadata values back to their original types
|
|
@@ -968,6 +990,123 @@ class PluginStorage {
|
|
|
968
990
|
if (!keyPrefix) return keys;
|
|
969
991
|
return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
|
|
970
992
|
}
|
|
993
|
+
/**
|
|
994
|
+
* Check if a key exists (not expired)
|
|
995
|
+
*
|
|
996
|
+
* @param {string} key - S3 key
|
|
997
|
+
* @returns {Promise<boolean>} True if exists and not expired
|
|
998
|
+
*/
|
|
999
|
+
async has(key) {
|
|
1000
|
+
const data = await this.get(key);
|
|
1001
|
+
return data !== null;
|
|
1002
|
+
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Check if a key is expired
|
|
1005
|
+
*
|
|
1006
|
+
* @param {string} key - S3 key
|
|
1007
|
+
* @returns {Promise<boolean>} True if expired or not found
|
|
1008
|
+
*/
|
|
1009
|
+
async isExpired(key) {
|
|
1010
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1011
|
+
if (!ok) {
|
|
1012
|
+
return true;
|
|
1013
|
+
}
|
|
1014
|
+
const metadata = response.Metadata || {};
|
|
1015
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1016
|
+
let data = parsedMetadata;
|
|
1017
|
+
if (response.Body) {
|
|
1018
|
+
try {
|
|
1019
|
+
const bodyContent = await response.Body.transformToString();
|
|
1020
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1021
|
+
const body = JSON.parse(bodyContent);
|
|
1022
|
+
data = { ...parsedMetadata, ...body };
|
|
1023
|
+
}
|
|
1024
|
+
} catch {
|
|
1025
|
+
return true;
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1029
|
+
if (!expiresAt) {
|
|
1030
|
+
return false;
|
|
1031
|
+
}
|
|
1032
|
+
return Date.now() > expiresAt;
|
|
1033
|
+
}
|
|
1034
|
+
/**
|
|
1035
|
+
* Get remaining TTL in seconds
|
|
1036
|
+
*
|
|
1037
|
+
* @param {string} key - S3 key
|
|
1038
|
+
* @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
|
|
1039
|
+
*/
|
|
1040
|
+
async getTTL(key) {
|
|
1041
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1042
|
+
if (!ok) {
|
|
1043
|
+
return null;
|
|
1044
|
+
}
|
|
1045
|
+
const metadata = response.Metadata || {};
|
|
1046
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1047
|
+
let data = parsedMetadata;
|
|
1048
|
+
if (response.Body) {
|
|
1049
|
+
try {
|
|
1050
|
+
const bodyContent = await response.Body.transformToString();
|
|
1051
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1052
|
+
const body = JSON.parse(bodyContent);
|
|
1053
|
+
data = { ...parsedMetadata, ...body };
|
|
1054
|
+
}
|
|
1055
|
+
} catch {
|
|
1056
|
+
return null;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1060
|
+
if (!expiresAt) {
|
|
1061
|
+
return null;
|
|
1062
|
+
}
|
|
1063
|
+
const remaining = Math.max(0, expiresAt - Date.now());
|
|
1064
|
+
return Math.floor(remaining / 1e3);
|
|
1065
|
+
}
|
|
1066
|
+
/**
|
|
1067
|
+
* Extend TTL by adding additional seconds
|
|
1068
|
+
*
|
|
1069
|
+
* @param {string} key - S3 key
|
|
1070
|
+
* @param {number} additionalSeconds - Seconds to add to current TTL
|
|
1071
|
+
* @returns {Promise<boolean>} True if extended, false if not found or no TTL
|
|
1072
|
+
*/
|
|
1073
|
+
async touch(key, additionalSeconds) {
|
|
1074
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1075
|
+
if (!ok) {
|
|
1076
|
+
return false;
|
|
1077
|
+
}
|
|
1078
|
+
const metadata = response.Metadata || {};
|
|
1079
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1080
|
+
let data = parsedMetadata;
|
|
1081
|
+
if (response.Body) {
|
|
1082
|
+
try {
|
|
1083
|
+
const bodyContent = await response.Body.transformToString();
|
|
1084
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1085
|
+
const body = JSON.parse(bodyContent);
|
|
1086
|
+
data = { ...parsedMetadata, ...body };
|
|
1087
|
+
}
|
|
1088
|
+
} catch {
|
|
1089
|
+
return false;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1093
|
+
if (!expiresAt) {
|
|
1094
|
+
return false;
|
|
1095
|
+
}
|
|
1096
|
+
data._expiresAt = expiresAt + additionalSeconds * 1e3;
|
|
1097
|
+
delete data._expiresat;
|
|
1098
|
+
const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, "body-overflow");
|
|
1099
|
+
const putParams = {
|
|
1100
|
+
key,
|
|
1101
|
+
metadata: newMetadata,
|
|
1102
|
+
contentType: "application/json"
|
|
1103
|
+
};
|
|
1104
|
+
if (newBody !== null) {
|
|
1105
|
+
putParams.body = JSON.stringify(newBody);
|
|
1106
|
+
}
|
|
1107
|
+
const [putOk] = await tryFn(() => this.client.putObject(putParams));
|
|
1108
|
+
return putOk;
|
|
1109
|
+
}
|
|
971
1110
|
/**
|
|
972
1111
|
* Delete a single object
|
|
973
1112
|
*
|
|
@@ -1045,6 +1184,78 @@ class PluginStorage {
|
|
|
1045
1184
|
}
|
|
1046
1185
|
return results;
|
|
1047
1186
|
}
|
|
1187
|
+
/**
|
|
1188
|
+
* Acquire a distributed lock with TTL and retry logic
|
|
1189
|
+
*
|
|
1190
|
+
* @param {string} lockName - Lock identifier
|
|
1191
|
+
* @param {Object} options - Lock options
|
|
1192
|
+
* @param {number} options.ttl - Lock TTL in seconds (default: 30)
|
|
1193
|
+
* @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
|
|
1194
|
+
* @param {string} options.workerId - Worker identifier (default: 'unknown')
|
|
1195
|
+
* @returns {Promise<Object|null>} Lock object or null if couldn't acquire
|
|
1196
|
+
*/
|
|
1197
|
+
async acquireLock(lockName, options = {}) {
|
|
1198
|
+
const { ttl = 30, timeout = 0, workerId = "unknown" } = options;
|
|
1199
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1200
|
+
const startTime = Date.now();
|
|
1201
|
+
while (true) {
|
|
1202
|
+
const existing = await this.get(key);
|
|
1203
|
+
if (!existing) {
|
|
1204
|
+
await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
|
|
1205
|
+
return { key, workerId };
|
|
1206
|
+
}
|
|
1207
|
+
if (Date.now() - startTime >= timeout) {
|
|
1208
|
+
return null;
|
|
1209
|
+
}
|
|
1210
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
/**
|
|
1214
|
+
* Release a distributed lock
|
|
1215
|
+
*
|
|
1216
|
+
* @param {string} lockName - Lock identifier
|
|
1217
|
+
* @returns {Promise<void>}
|
|
1218
|
+
*/
|
|
1219
|
+
async releaseLock(lockName) {
|
|
1220
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1221
|
+
await this.delete(key);
|
|
1222
|
+
}
|
|
1223
|
+
/**
|
|
1224
|
+
* Check if a lock is currently held
|
|
1225
|
+
*
|
|
1226
|
+
* @param {string} lockName - Lock identifier
|
|
1227
|
+
* @returns {Promise<boolean>} True if locked
|
|
1228
|
+
*/
|
|
1229
|
+
async isLocked(lockName) {
|
|
1230
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1231
|
+
const lock = await this.get(key);
|
|
1232
|
+
return lock !== null;
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Increment a counter value
|
|
1236
|
+
*
|
|
1237
|
+
* @param {string} key - S3 key
|
|
1238
|
+
* @param {number} amount - Amount to increment (default: 1)
|
|
1239
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1240
|
+
* @returns {Promise<number>} New value
|
|
1241
|
+
*/
|
|
1242
|
+
async increment(key, amount = 1, options = {}) {
|
|
1243
|
+
const data = await this.get(key);
|
|
1244
|
+
const value = (data?.value || 0) + amount;
|
|
1245
|
+
await this.set(key, { value }, options);
|
|
1246
|
+
return value;
|
|
1247
|
+
}
|
|
1248
|
+
/**
|
|
1249
|
+
* Decrement a counter value
|
|
1250
|
+
*
|
|
1251
|
+
* @param {string} key - S3 key
|
|
1252
|
+
* @param {number} amount - Amount to decrement (default: 1)
|
|
1253
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1254
|
+
* @returns {Promise<number>} New value
|
|
1255
|
+
*/
|
|
1256
|
+
async decrement(key, amount = 1, options = {}) {
|
|
1257
|
+
return this.increment(key, -amount, options);
|
|
1258
|
+
}
|
|
1048
1259
|
/**
|
|
1049
1260
|
* Apply behavior to split data between metadata and body
|
|
1050
1261
|
*
|
|
@@ -1348,12 +1559,18 @@ class AuditPlugin extends Plugin {
|
|
|
1348
1559
|
recordId: "string|required",
|
|
1349
1560
|
userId: "string|optional",
|
|
1350
1561
|
timestamp: "string|required",
|
|
1562
|
+
createdAt: "string|required",
|
|
1563
|
+
// YYYY-MM-DD for partitioning
|
|
1351
1564
|
oldData: "string|optional",
|
|
1352
1565
|
newData: "string|optional",
|
|
1353
1566
|
partition: "string|optional",
|
|
1354
1567
|
partitionValues: "string|optional",
|
|
1355
1568
|
metadata: "string|optional"
|
|
1356
1569
|
},
|
|
1570
|
+
partitions: {
|
|
1571
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } },
|
|
1572
|
+
byResource: { fields: { resourceName: "string" } }
|
|
1573
|
+
},
|
|
1357
1574
|
behavior: "body-overflow"
|
|
1358
1575
|
}));
|
|
1359
1576
|
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
@@ -1457,10 +1674,13 @@ class AuditPlugin extends Plugin {
|
|
|
1457
1674
|
if (!this.auditResource) {
|
|
1458
1675
|
return;
|
|
1459
1676
|
}
|
|
1677
|
+
const now = /* @__PURE__ */ new Date();
|
|
1460
1678
|
const auditRecord = {
|
|
1461
1679
|
id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
|
|
1462
1680
|
userId: this.getCurrentUserId?.() || "system",
|
|
1463
|
-
timestamp:
|
|
1681
|
+
timestamp: now.toISOString(),
|
|
1682
|
+
createdAt: now.toISOString().slice(0, 10),
|
|
1683
|
+
// YYYY-MM-DD for partitioning
|
|
1464
1684
|
metadata: JSON.stringify({ source: "audit-plugin", version: "2.0" }),
|
|
1465
1685
|
resourceName: auditData.resourceName,
|
|
1466
1686
|
operation: auditData.operation,
|
|
@@ -1535,9 +1755,25 @@ class AuditPlugin extends Plugin {
|
|
|
1535
1755
|
async getAuditLogs(options = {}) {
|
|
1536
1756
|
if (!this.auditResource) return [];
|
|
1537
1757
|
const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
|
|
1538
|
-
const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
|
|
1539
1758
|
let items = [];
|
|
1540
|
-
if (
|
|
1759
|
+
if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
|
|
1760
|
+
const [ok, err, result] = await tryFn(
|
|
1761
|
+
() => this.auditResource.query({ resourceName }, { limit: limit + offset })
|
|
1762
|
+
);
|
|
1763
|
+
items = ok && result ? result : [];
|
|
1764
|
+
return items.slice(offset, offset + limit);
|
|
1765
|
+
} else if (startDate && !resourceName && !operation && !recordId && !partition) {
|
|
1766
|
+
const dates = this._generateDateRange(startDate, endDate);
|
|
1767
|
+
for (const date of dates) {
|
|
1768
|
+
const [ok, err, result] = await tryFn(
|
|
1769
|
+
() => this.auditResource.query({ createdAt: date })
|
|
1770
|
+
);
|
|
1771
|
+
if (ok && result) {
|
|
1772
|
+
items.push(...result);
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
return items.slice(offset, offset + limit);
|
|
1776
|
+
} else if (resourceName || operation || recordId || partition || startDate || endDate) {
|
|
1541
1777
|
const fetchSize = Math.min(1e4, Math.max(1e3, (limit + offset) * 20));
|
|
1542
1778
|
const result = await this.auditResource.list({ limit: fetchSize });
|
|
1543
1779
|
items = result || [];
|
|
@@ -1567,6 +1803,15 @@ class AuditPlugin extends Plugin {
|
|
|
1567
1803
|
return result.items || [];
|
|
1568
1804
|
}
|
|
1569
1805
|
}
|
|
1806
|
+
_generateDateRange(startDate, endDate) {
|
|
1807
|
+
const dates = [];
|
|
1808
|
+
const start = new Date(startDate);
|
|
1809
|
+
const end = endDate ? new Date(endDate) : /* @__PURE__ */ new Date();
|
|
1810
|
+
for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
|
|
1811
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
1812
|
+
}
|
|
1813
|
+
return dates;
|
|
1814
|
+
}
|
|
1570
1815
|
async getRecordHistory(resourceName, recordId) {
|
|
1571
1816
|
return await this.getAuditLogs({ resourceName, recordId });
|
|
1572
1817
|
}
|
|
@@ -1599,6 +1844,37 @@ class AuditPlugin extends Plugin {
|
|
|
1599
1844
|
}
|
|
1600
1845
|
return stats;
|
|
1601
1846
|
}
|
|
1847
|
+
/**
|
|
1848
|
+
* Clean up audit logs older than retention period
|
|
1849
|
+
* @param {number} retentionDays - Number of days to retain (default: 90)
|
|
1850
|
+
* @returns {Promise<number>} Number of records deleted
|
|
1851
|
+
*/
|
|
1852
|
+
async cleanupOldAudits(retentionDays = 90) {
|
|
1853
|
+
if (!this.auditResource) return 0;
|
|
1854
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
1855
|
+
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
|
1856
|
+
const datesToDelete = [];
|
|
1857
|
+
const startDate = new Date(cutoffDate);
|
|
1858
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
1859
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
1860
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
1861
|
+
}
|
|
1862
|
+
let deletedCount = 0;
|
|
1863
|
+
for (const dateStr of datesToDelete) {
|
|
1864
|
+
const [ok, err, oldAudits] = await tryFn(
|
|
1865
|
+
() => this.auditResource.query({ createdAt: dateStr })
|
|
1866
|
+
);
|
|
1867
|
+
if (ok && oldAudits) {
|
|
1868
|
+
for (const audit of oldAudits) {
|
|
1869
|
+
const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
|
|
1870
|
+
if (delOk) {
|
|
1871
|
+
deletedCount++;
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
return deletedCount;
|
|
1877
|
+
}
|
|
1602
1878
|
}
|
|
1603
1879
|
|
|
1604
1880
|
class BaseBackupDriver {
|
|
@@ -4834,6 +5110,8 @@ function createConfig(options, detectedTimezone) {
|
|
|
4834
5110
|
consolidationWindow: consolidation.window ?? 24,
|
|
4835
5111
|
autoConsolidate: consolidation.auto !== false,
|
|
4836
5112
|
mode: consolidation.mode || "async",
|
|
5113
|
+
// ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
|
|
5114
|
+
markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
|
|
4837
5115
|
// Late arrivals
|
|
4838
5116
|
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
4839
5117
|
// Batch transactions
|
|
@@ -4860,9 +5138,7 @@ function createConfig(options, detectedTimezone) {
|
|
|
4860
5138
|
deleteConsolidatedTransactions: checkpoints.deleteConsolidated !== false,
|
|
4861
5139
|
autoCheckpoint: checkpoints.auto !== false,
|
|
4862
5140
|
// Debug
|
|
4863
|
-
verbose: options.verbose
|
|
4864
|
-
// Default: true (can disable with verbose: false)
|
|
4865
|
-
debug: options.debug || false
|
|
5141
|
+
verbose: options.verbose || false
|
|
4866
5142
|
};
|
|
4867
5143
|
}
|
|
4868
5144
|
function validateResourcesConfig(resources) {
|
|
@@ -4939,6 +5215,21 @@ function getTimezoneOffset(timezone, verbose = false) {
|
|
|
4939
5215
|
return offsets[timezone] || 0;
|
|
4940
5216
|
}
|
|
4941
5217
|
}
|
|
5218
|
+
function getISOWeek(date) {
|
|
5219
|
+
const target = new Date(date.valueOf());
|
|
5220
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
5221
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
5222
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
5223
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
5224
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
5225
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
5226
|
+
}
|
|
5227
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
5228
|
+
return {
|
|
5229
|
+
year: target.getUTCFullYear(),
|
|
5230
|
+
week: weekNumber
|
|
5231
|
+
};
|
|
5232
|
+
}
|
|
4942
5233
|
function getCohortInfo(date, timezone, verbose = false) {
|
|
4943
5234
|
const offset = getTimezoneOffset(timezone, verbose);
|
|
4944
5235
|
const localDate = new Date(date.getTime() + offset);
|
|
@@ -4946,10 +5237,14 @@ function getCohortInfo(date, timezone, verbose = false) {
|
|
|
4946
5237
|
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4947
5238
|
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4948
5239
|
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
5240
|
+
const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
|
|
5241
|
+
const week = `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
4949
5242
|
return {
|
|
4950
5243
|
date: `${year}-${month}-${day}`,
|
|
4951
5244
|
hour: `${year}-${month}-${day}T${hour}`,
|
|
4952
5245
|
// ISO-like format for hour partition
|
|
5246
|
+
week,
|
|
5247
|
+
// ISO 8601 week format (e.g., '2025-W42')
|
|
4953
5248
|
month: `${year}-${month}`
|
|
4954
5249
|
};
|
|
4955
5250
|
}
|
|
@@ -5027,6 +5322,11 @@ function createPartitionConfig() {
|
|
|
5027
5322
|
cohortDate: "string"
|
|
5028
5323
|
}
|
|
5029
5324
|
},
|
|
5325
|
+
byWeek: {
|
|
5326
|
+
fields: {
|
|
5327
|
+
cohortWeek: "string"
|
|
5328
|
+
}
|
|
5329
|
+
},
|
|
5030
5330
|
byMonth: {
|
|
5031
5331
|
fields: {
|
|
5032
5332
|
cohortMonth: "string"
|
|
@@ -5066,6 +5366,7 @@ async function createTransaction(handler, data, config) {
|
|
|
5066
5366
|
timestamp: now.toISOString(),
|
|
5067
5367
|
cohortDate: cohortInfo.date,
|
|
5068
5368
|
cohortHour: cohortInfo.hour,
|
|
5369
|
+
cohortWeek: cohortInfo.week,
|
|
5069
5370
|
cohortMonth: cohortInfo.month,
|
|
5070
5371
|
source: data.source || "unknown",
|
|
5071
5372
|
applied: false
|
|
@@ -5106,50 +5407,6 @@ async function flushPendingTransactions(handler) {
|
|
|
5106
5407
|
}
|
|
5107
5408
|
}
|
|
5108
5409
|
|
|
5109
|
-
async function cleanupStaleLocks(lockResource, config) {
|
|
5110
|
-
const now = Date.now();
|
|
5111
|
-
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
5112
|
-
const cutoffTime = now - lockTimeoutMs;
|
|
5113
|
-
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
5114
|
-
const [lockAcquired] = await tryFn(
|
|
5115
|
-
() => lockResource.insert({
|
|
5116
|
-
id: cleanupLockId,
|
|
5117
|
-
lockedAt: Date.now(),
|
|
5118
|
-
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5119
|
-
})
|
|
5120
|
-
);
|
|
5121
|
-
if (!lockAcquired) {
|
|
5122
|
-
if (config.verbose) {
|
|
5123
|
-
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
5124
|
-
}
|
|
5125
|
-
return;
|
|
5126
|
-
}
|
|
5127
|
-
try {
|
|
5128
|
-
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
5129
|
-
if (!ok || !locks || locks.length === 0) return;
|
|
5130
|
-
const staleLocks = locks.filter(
|
|
5131
|
-
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
5132
|
-
);
|
|
5133
|
-
if (staleLocks.length === 0) return;
|
|
5134
|
-
if (config.verbose) {
|
|
5135
|
-
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
5136
|
-
}
|
|
5137
|
-
const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
5138
|
-
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
5139
|
-
return deleted;
|
|
5140
|
-
});
|
|
5141
|
-
if (errors && errors.length > 0 && config.verbose) {
|
|
5142
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5143
|
-
}
|
|
5144
|
-
} catch (error) {
|
|
5145
|
-
if (config.verbose) {
|
|
5146
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5147
|
-
}
|
|
5148
|
-
} finally {
|
|
5149
|
-
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
5150
|
-
}
|
|
5151
|
-
}
|
|
5152
|
-
|
|
5153
5410
|
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
5154
5411
|
const intervalMs = config.consolidationInterval * 1e3;
|
|
5155
5412
|
if (config.verbose) {
|
|
@@ -5246,17 +5503,15 @@ async function runConsolidation(transactionResource, consolidateRecordFn, emitFn
|
|
|
5246
5503
|
}
|
|
5247
5504
|
}
|
|
5248
5505
|
}
|
|
5249
|
-
async function consolidateRecord(originalId, transactionResource, targetResource,
|
|
5250
|
-
|
|
5251
|
-
const
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
);
|
|
5259
|
-
if (!lockAcquired) {
|
|
5506
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, storage, analyticsResource, updateAnalyticsFn, config) {
|
|
5507
|
+
const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
|
|
5508
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5509
|
+
ttl: config.lockTimeout || 30,
|
|
5510
|
+
timeout: 0,
|
|
5511
|
+
// Don't wait if locked
|
|
5512
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5513
|
+
});
|
|
5514
|
+
if (!lock) {
|
|
5260
5515
|
if (config.verbose) {
|
|
5261
5516
|
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
5262
5517
|
}
|
|
@@ -5401,7 +5656,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5401
5656
|
`[EventualConsistency] ${config.resource}.${config.field} - ${originalId}: ${currentValue} \u2192 ${consolidatedValue} (${consolidatedValue > currentValue ? "+" : ""}${consolidatedValue - currentValue})`
|
|
5402
5657
|
);
|
|
5403
5658
|
}
|
|
5404
|
-
if (config.
|
|
5659
|
+
if (config.verbose) {
|
|
5405
5660
|
console.log(
|
|
5406
5661
|
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5407
5662
|
originalId: '${originalId}',
|
|
@@ -5416,7 +5671,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5416
5671
|
[config.field]: consolidatedValue
|
|
5417
5672
|
})
|
|
5418
5673
|
);
|
|
5419
|
-
if (config.
|
|
5674
|
+
if (config.verbose) {
|
|
5420
5675
|
console.log(
|
|
5421
5676
|
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5422
5677
|
updateOk: ${updateOk},
|
|
@@ -5426,7 +5681,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5426
5681
|
}`
|
|
5427
5682
|
);
|
|
5428
5683
|
}
|
|
5429
|
-
if (updateOk &&
|
|
5684
|
+
if (updateOk && config.verbose) {
|
|
5430
5685
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5431
5686
|
() => targetResource.get(originalId, { skipCache: true })
|
|
5432
5687
|
);
|
|
@@ -5467,7 +5722,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5467
5722
|
}
|
|
5468
5723
|
if (updateOk) {
|
|
5469
5724
|
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5470
|
-
const
|
|
5725
|
+
const markAppliedConcurrency = config.markAppliedConcurrency || 50;
|
|
5726
|
+
const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
|
|
5471
5727
|
const [ok2, err2] = await tryFn(
|
|
5472
5728
|
() => transactionResource.update(txn.id, { applied: true })
|
|
5473
5729
|
);
|
|
@@ -5515,9 +5771,11 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5515
5771
|
}
|
|
5516
5772
|
return consolidatedValue;
|
|
5517
5773
|
} finally {
|
|
5518
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5774
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5775
|
+
() => storage.releaseLock(lockKey)
|
|
5776
|
+
);
|
|
5519
5777
|
if (!lockReleased && config.verbose) {
|
|
5520
|
-
console.warn(`[EventualConsistency] Failed to release lock ${
|
|
5778
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5521
5779
|
}
|
|
5522
5780
|
}
|
|
5523
5781
|
}
|
|
@@ -5591,17 +5849,15 @@ async function getCohortStats(cohortDate, transactionResource) {
|
|
|
5591
5849
|
}
|
|
5592
5850
|
return stats;
|
|
5593
5851
|
}
|
|
5594
|
-
async function recalculateRecord(originalId, transactionResource, targetResource,
|
|
5595
|
-
|
|
5596
|
-
const
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
);
|
|
5604
|
-
if (!lockAcquired) {
|
|
5852
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, storage, consolidateRecordFn, config) {
|
|
5853
|
+
const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
|
|
5854
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5855
|
+
ttl: config.lockTimeout || 30,
|
|
5856
|
+
timeout: 0,
|
|
5857
|
+
// Don't wait if locked
|
|
5858
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5859
|
+
});
|
|
5860
|
+
if (!lock) {
|
|
5605
5861
|
if (config.verbose) {
|
|
5606
5862
|
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5607
5863
|
}
|
|
@@ -5669,9 +5925,11 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5669
5925
|
}
|
|
5670
5926
|
return consolidatedValue;
|
|
5671
5927
|
} finally {
|
|
5672
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5928
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5929
|
+
() => storage.releaseLock(lockKey)
|
|
5930
|
+
);
|
|
5673
5931
|
if (!lockReleased && config.verbose) {
|
|
5674
|
-
console.warn(`[EventualConsistency] Failed to release recalculate lock ${
|
|
5932
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5675
5933
|
}
|
|
5676
5934
|
}
|
|
5677
5935
|
}
|
|
@@ -5683,16 +5941,16 @@ function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCall
|
|
|
5683
5941
|
}, gcIntervalMs);
|
|
5684
5942
|
return handler.gcTimer;
|
|
5685
5943
|
}
|
|
5686
|
-
async function runGarbageCollection(transactionResource,
|
|
5687
|
-
const
|
|
5688
|
-
const
|
|
5689
|
-
|
|
5690
|
-
|
|
5691
|
-
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
);
|
|
5695
|
-
if (!
|
|
5944
|
+
async function runGarbageCollection(transactionResource, storage, config, emitFn) {
|
|
5945
|
+
const lockKey = `gc-${config.resource}-${config.field}`;
|
|
5946
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5947
|
+
ttl: 300,
|
|
5948
|
+
// 5 minutes for GC
|
|
5949
|
+
timeout: 0,
|
|
5950
|
+
// Don't wait if locked
|
|
5951
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5952
|
+
});
|
|
5953
|
+
if (!lock) {
|
|
5696
5954
|
if (config.verbose) {
|
|
5697
5955
|
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5698
5956
|
}
|
|
@@ -5750,7 +6008,7 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
|
|
|
5750
6008
|
emitFn("eventual-consistency.gc-error", error);
|
|
5751
6009
|
}
|
|
5752
6010
|
} finally {
|
|
5753
|
-
await tryFn(() =>
|
|
6011
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
5754
6012
|
}
|
|
5755
6013
|
}
|
|
5756
6014
|
|
|
@@ -5760,12 +6018,12 @@ async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
5760
6018
|
throw new Error(
|
|
5761
6019
|
`[EventualConsistency] CRITICAL BUG: config.field is undefined in updateAnalytics()!
|
|
5762
6020
|
This indicates a race condition in the plugin where multiple handlers are sharing the same config object.
|
|
5763
|
-
Config: ${JSON.stringify({ resource: config.resource, field: config.field
|
|
6021
|
+
Config: ${JSON.stringify({ resource: config.resource, field: config.field })}
|
|
5764
6022
|
Transactions count: ${transactions.length}
|
|
5765
6023
|
AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
5766
6024
|
);
|
|
5767
6025
|
}
|
|
5768
|
-
if (config.verbose
|
|
6026
|
+
if (config.verbose) {
|
|
5769
6027
|
console.log(
|
|
5770
6028
|
`[EventualConsistency] ${config.resource}.${config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5771
6029
|
);
|
|
@@ -5773,26 +6031,30 @@ AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
|
5773
6031
|
try {
|
|
5774
6032
|
const byHour = groupByCohort(transactions, "cohortHour");
|
|
5775
6033
|
const cohortCount = Object.keys(byHour).length;
|
|
5776
|
-
if (config.verbose
|
|
6034
|
+
if (config.verbose) {
|
|
5777
6035
|
console.log(
|
|
5778
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
6036
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
|
|
5779
6037
|
);
|
|
5780
6038
|
}
|
|
5781
|
-
|
|
5782
|
-
|
|
5783
|
-
|
|
6039
|
+
await Promise.all(
|
|
6040
|
+
Object.entries(byHour).map(
|
|
6041
|
+
([cohort, txns]) => upsertAnalytics("hour", cohort, txns, analyticsResource, config)
|
|
6042
|
+
)
|
|
6043
|
+
);
|
|
5784
6044
|
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5785
6045
|
const uniqueHours = Object.keys(byHour);
|
|
5786
|
-
if (config.verbose
|
|
6046
|
+
if (config.verbose) {
|
|
5787
6047
|
console.log(
|
|
5788
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
6048
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
|
|
5789
6049
|
);
|
|
5790
6050
|
}
|
|
5791
|
-
|
|
5792
|
-
|
|
5793
|
-
|
|
6051
|
+
await Promise.all(
|
|
6052
|
+
uniqueHours.map(
|
|
6053
|
+
(cohortHour) => rollupAnalytics(cohortHour, analyticsResource, config)
|
|
6054
|
+
)
|
|
6055
|
+
);
|
|
5794
6056
|
}
|
|
5795
|
-
if (config.verbose
|
|
6057
|
+
if (config.verbose) {
|
|
5796
6058
|
console.log(
|
|
5797
6059
|
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5798
6060
|
);
|
|
@@ -5893,18 +6155,53 @@ function calculateOperationBreakdown(transactions) {
|
|
|
5893
6155
|
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5894
6156
|
const cohortDate = cohortHour.substring(0, 10);
|
|
5895
6157
|
const cohortMonth = cohortHour.substring(0, 7);
|
|
6158
|
+
const date = new Date(cohortDate);
|
|
6159
|
+
const cohortWeek = getCohortWeekFromDate(date);
|
|
5896
6160
|
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
|
|
6161
|
+
await rollupPeriod("week", cohortWeek, cohortWeek, analyticsResource, config);
|
|
5897
6162
|
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
|
|
5898
6163
|
}
|
|
6164
|
+
function getCohortWeekFromDate(date) {
|
|
6165
|
+
const target = new Date(date.valueOf());
|
|
6166
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
6167
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
6168
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
6169
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
6170
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
6171
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
6172
|
+
}
|
|
6173
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
6174
|
+
const weekYear = target.getUTCFullYear();
|
|
6175
|
+
return `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
6176
|
+
}
|
|
5899
6177
|
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5900
|
-
|
|
6178
|
+
let sourcePeriod;
|
|
6179
|
+
if (period === "day") {
|
|
6180
|
+
sourcePeriod = "hour";
|
|
6181
|
+
} else if (period === "week") {
|
|
6182
|
+
sourcePeriod = "day";
|
|
6183
|
+
} else if (period === "month") {
|
|
6184
|
+
sourcePeriod = "week";
|
|
6185
|
+
} else {
|
|
6186
|
+
sourcePeriod = "day";
|
|
6187
|
+
}
|
|
5901
6188
|
const [ok, err, allAnalytics] = await tryFn(
|
|
5902
6189
|
() => analyticsResource.list()
|
|
5903
6190
|
);
|
|
5904
6191
|
if (!ok || !allAnalytics) return;
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
6192
|
+
let sourceAnalytics;
|
|
6193
|
+
if (period === "week") {
|
|
6194
|
+
sourceAnalytics = allAnalytics.filter((a) => {
|
|
6195
|
+
if (a.period !== sourcePeriod) return false;
|
|
6196
|
+
const dayDate = new Date(a.cohort);
|
|
6197
|
+
const dayWeek = getCohortWeekFromDate(dayDate);
|
|
6198
|
+
return dayWeek === cohort;
|
|
6199
|
+
});
|
|
6200
|
+
} else {
|
|
6201
|
+
sourceAnalytics = allAnalytics.filter(
|
|
6202
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
6203
|
+
);
|
|
6204
|
+
}
|
|
5908
6205
|
if (sourceAnalytics.length === 0) return;
|
|
5909
6206
|
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5910
6207
|
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
@@ -6113,6 +6410,32 @@ async function getYearByMonth(resourceName, field, year, options, fieldHandlers)
|
|
|
6113
6410
|
}
|
|
6114
6411
|
return data;
|
|
6115
6412
|
}
|
|
6413
|
+
async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
|
|
6414
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6415
|
+
period: "week",
|
|
6416
|
+
year
|
|
6417
|
+
}, fieldHandlers);
|
|
6418
|
+
if (options.fillGaps) {
|
|
6419
|
+
const startWeek = `${year}-W01`;
|
|
6420
|
+
const endWeek = `${year}-W53`;
|
|
6421
|
+
return fillGaps(data, "week", startWeek, endWeek);
|
|
6422
|
+
}
|
|
6423
|
+
return data;
|
|
6424
|
+
}
|
|
6425
|
+
async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
|
|
6426
|
+
const year = parseInt(month.substring(0, 4));
|
|
6427
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
6428
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
6429
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
6430
|
+
const firstWeek = getCohortWeekFromDate(firstDay);
|
|
6431
|
+
const lastWeek = getCohortWeekFromDate(lastDay);
|
|
6432
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6433
|
+
period: "week",
|
|
6434
|
+
startDate: firstWeek,
|
|
6435
|
+
endDate: lastWeek
|
|
6436
|
+
}, fieldHandlers);
|
|
6437
|
+
return data;
|
|
6438
|
+
}
|
|
6116
6439
|
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
6117
6440
|
let year, monthNum;
|
|
6118
6441
|
if (month === "last") {
|
|
@@ -6340,7 +6663,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6340
6663
|
if (!handler.targetResource) return;
|
|
6341
6664
|
const resourceName = handler.resource;
|
|
6342
6665
|
const fieldName = handler.field;
|
|
6343
|
-
const transactionResourceName =
|
|
6666
|
+
const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
|
|
6344
6667
|
const partitionConfig = createPartitionConfig();
|
|
6345
6668
|
const [ok, err, transactionResource] = await tryFn(
|
|
6346
6669
|
() => database.createResource({
|
|
@@ -6354,6 +6677,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6354
6677
|
timestamp: "string|required",
|
|
6355
6678
|
cohortDate: "string|required",
|
|
6356
6679
|
cohortHour: "string|required",
|
|
6680
|
+
cohortWeek: "string|optional",
|
|
6357
6681
|
cohortMonth: "string|optional",
|
|
6358
6682
|
source: "string|optional",
|
|
6359
6683
|
applied: "boolean|optional"
|
|
@@ -6369,36 +6693,18 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6369
6693
|
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
6370
6694
|
}
|
|
6371
6695
|
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
6372
|
-
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
6373
|
-
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
6374
|
-
() => database.createResource({
|
|
6375
|
-
name: lockResourceName,
|
|
6376
|
-
attributes: {
|
|
6377
|
-
id: "string|required",
|
|
6378
|
-
lockedAt: "number|required",
|
|
6379
|
-
workerId: "string|optional"
|
|
6380
|
-
},
|
|
6381
|
-
behavior: "body-only",
|
|
6382
|
-
timestamps: false,
|
|
6383
|
-
createdBy: "EventualConsistencyPlugin"
|
|
6384
|
-
})
|
|
6385
|
-
);
|
|
6386
|
-
if (!lockOk && !database.resources[lockResourceName]) {
|
|
6387
|
-
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
6388
|
-
}
|
|
6389
|
-
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
6390
6696
|
if (config.enableAnalytics) {
|
|
6391
6697
|
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
6392
6698
|
}
|
|
6393
6699
|
addHelperMethodsForHandler(handler, plugin, config);
|
|
6394
6700
|
if (config.verbose) {
|
|
6395
6701
|
console.log(
|
|
6396
|
-
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}
|
|
6702
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ""} (locks via PluginStorage TTL)`
|
|
6397
6703
|
);
|
|
6398
6704
|
}
|
|
6399
6705
|
}
|
|
6400
6706
|
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
6401
|
-
const analyticsResourceName =
|
|
6707
|
+
const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
|
|
6402
6708
|
const [ok, err, analyticsResource] = await tryFn(
|
|
6403
6709
|
() => database.createResource({
|
|
6404
6710
|
name: analyticsResourceName,
|
|
@@ -6572,7 +6878,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6572
6878
|
originalId,
|
|
6573
6879
|
this.transactionResource,
|
|
6574
6880
|
this.targetResource,
|
|
6575
|
-
this.
|
|
6881
|
+
this.getStorage(),
|
|
6576
6882
|
this.analyticsResource,
|
|
6577
6883
|
(transactions) => this.updateAnalytics(transactions),
|
|
6578
6884
|
this.config
|
|
@@ -6608,7 +6914,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6608
6914
|
originalId,
|
|
6609
6915
|
this.transactionResource,
|
|
6610
6916
|
this.targetResource,
|
|
6611
|
-
this.
|
|
6917
|
+
this.getStorage(),
|
|
6612
6918
|
(id) => this.consolidateRecord(id),
|
|
6613
6919
|
this.config
|
|
6614
6920
|
);
|
|
@@ -6629,20 +6935,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6629
6935
|
const oldField = this.config.field;
|
|
6630
6936
|
const oldTransactionResource = this.transactionResource;
|
|
6631
6937
|
const oldTargetResource = this.targetResource;
|
|
6632
|
-
const oldLockResource = this.lockResource;
|
|
6633
6938
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6634
6939
|
this.config.resource = handler.resource;
|
|
6635
6940
|
this.config.field = handler.field;
|
|
6636
6941
|
this.transactionResource = handler.transactionResource;
|
|
6637
6942
|
this.targetResource = handler.targetResource;
|
|
6638
|
-
this.lockResource = handler.lockResource;
|
|
6639
6943
|
this.analyticsResource = handler.analyticsResource;
|
|
6640
6944
|
const result = await this.consolidateRecord(id);
|
|
6641
6945
|
this.config.resource = oldResource;
|
|
6642
6946
|
this.config.field = oldField;
|
|
6643
6947
|
this.transactionResource = oldTransactionResource;
|
|
6644
6948
|
this.targetResource = oldTargetResource;
|
|
6645
|
-
this.lockResource = oldLockResource;
|
|
6646
6949
|
this.analyticsResource = oldAnalyticsResource;
|
|
6647
6950
|
return result;
|
|
6648
6951
|
}
|
|
@@ -6655,20 +6958,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6655
6958
|
const oldField = this.config.field;
|
|
6656
6959
|
const oldTransactionResource = this.transactionResource;
|
|
6657
6960
|
const oldTargetResource = this.targetResource;
|
|
6658
|
-
const oldLockResource = this.lockResource;
|
|
6659
6961
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6660
6962
|
this.config.resource = handler.resource;
|
|
6661
6963
|
this.config.field = handler.field;
|
|
6662
6964
|
this.transactionResource = handler.transactionResource;
|
|
6663
6965
|
this.targetResource = handler.targetResource;
|
|
6664
|
-
this.lockResource = handler.lockResource;
|
|
6665
6966
|
this.analyticsResource = handler.analyticsResource;
|
|
6666
6967
|
const result = await this.consolidateRecord(id);
|
|
6667
6968
|
this.config.resource = oldResource;
|
|
6668
6969
|
this.config.field = oldField;
|
|
6669
6970
|
this.transactionResource = oldTransactionResource;
|
|
6670
6971
|
this.targetResource = oldTargetResource;
|
|
6671
|
-
this.lockResource = oldLockResource;
|
|
6672
6972
|
this.analyticsResource = oldAnalyticsResource;
|
|
6673
6973
|
return result;
|
|
6674
6974
|
}
|
|
@@ -6701,20 +7001,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6701
7001
|
const oldField = this.config.field;
|
|
6702
7002
|
const oldTransactionResource = this.transactionResource;
|
|
6703
7003
|
const oldTargetResource = this.targetResource;
|
|
6704
|
-
const oldLockResource = this.lockResource;
|
|
6705
7004
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6706
7005
|
this.config.resource = handler.resource;
|
|
6707
7006
|
this.config.field = handler.field;
|
|
6708
7007
|
this.transactionResource = handler.transactionResource;
|
|
6709
7008
|
this.targetResource = handler.targetResource;
|
|
6710
|
-
this.lockResource = handler.lockResource;
|
|
6711
7009
|
this.analyticsResource = handler.analyticsResource;
|
|
6712
7010
|
const result = await this.recalculateRecord(id);
|
|
6713
7011
|
this.config.resource = oldResource;
|
|
6714
7012
|
this.config.field = oldField;
|
|
6715
7013
|
this.transactionResource = oldTransactionResource;
|
|
6716
7014
|
this.targetResource = oldTargetResource;
|
|
6717
|
-
this.lockResource = oldLockResource;
|
|
6718
7015
|
this.analyticsResource = oldAnalyticsResource;
|
|
6719
7016
|
return result;
|
|
6720
7017
|
}
|
|
@@ -6727,13 +7024,11 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6727
7024
|
const oldField = this.config.field;
|
|
6728
7025
|
const oldTransactionResource = this.transactionResource;
|
|
6729
7026
|
const oldTargetResource = this.targetResource;
|
|
6730
|
-
const oldLockResource = this.lockResource;
|
|
6731
7027
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6732
7028
|
this.config.resource = resourceName;
|
|
6733
7029
|
this.config.field = fieldName;
|
|
6734
7030
|
this.transactionResource = handler.transactionResource;
|
|
6735
7031
|
this.targetResource = handler.targetResource;
|
|
6736
|
-
this.lockResource = handler.lockResource;
|
|
6737
7032
|
this.analyticsResource = handler.analyticsResource;
|
|
6738
7033
|
try {
|
|
6739
7034
|
await runConsolidation(
|
|
@@ -6747,7 +7042,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6747
7042
|
this.config.field = oldField;
|
|
6748
7043
|
this.transactionResource = oldTransactionResource;
|
|
6749
7044
|
this.targetResource = oldTargetResource;
|
|
6750
|
-
this.lockResource = oldLockResource;
|
|
6751
7045
|
this.analyticsResource = oldAnalyticsResource;
|
|
6752
7046
|
}
|
|
6753
7047
|
}
|
|
@@ -6760,16 +7054,14 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6760
7054
|
const oldField = this.config.field;
|
|
6761
7055
|
const oldTransactionResource = this.transactionResource;
|
|
6762
7056
|
const oldTargetResource = this.targetResource;
|
|
6763
|
-
const oldLockResource = this.lockResource;
|
|
6764
7057
|
this.config.resource = resourceName;
|
|
6765
7058
|
this.config.field = fieldName;
|
|
6766
7059
|
this.transactionResource = handler.transactionResource;
|
|
6767
7060
|
this.targetResource = handler.targetResource;
|
|
6768
|
-
this.lockResource = handler.lockResource;
|
|
6769
7061
|
try {
|
|
6770
7062
|
await runGarbageCollection(
|
|
6771
7063
|
this.transactionResource,
|
|
6772
|
-
this.
|
|
7064
|
+
this.getStorage(),
|
|
6773
7065
|
this.config,
|
|
6774
7066
|
(event, data) => this.emit(event, data)
|
|
6775
7067
|
);
|
|
@@ -6778,7 +7070,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6778
7070
|
this.config.field = oldField;
|
|
6779
7071
|
this.transactionResource = oldTransactionResource;
|
|
6780
7072
|
this.targetResource = oldTargetResource;
|
|
6781
|
-
this.lockResource = oldLockResource;
|
|
6782
7073
|
}
|
|
6783
7074
|
}
|
|
6784
7075
|
// Public Analytics API
|
|
@@ -6847,6 +7138,28 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6847
7138
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
6848
7139
|
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6849
7140
|
}
|
|
7141
|
+
/**
|
|
7142
|
+
* Get analytics for entire year, broken down by weeks
|
|
7143
|
+
* @param {string} resourceName - Resource name
|
|
7144
|
+
* @param {string} field - Field name
|
|
7145
|
+
* @param {number} year - Year (e.g., 2025)
|
|
7146
|
+
* @param {Object} options - Options
|
|
7147
|
+
* @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
|
|
7148
|
+
*/
|
|
7149
|
+
async getYearByWeek(resourceName, field, year, options = {}) {
|
|
7150
|
+
return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
|
|
7151
|
+
}
|
|
7152
|
+
/**
|
|
7153
|
+
* Get analytics for entire month, broken down by weeks
|
|
7154
|
+
* @param {string} resourceName - Resource name
|
|
7155
|
+
* @param {string} field - Field name
|
|
7156
|
+
* @param {string} month - Month in YYYY-MM format
|
|
7157
|
+
* @param {Object} options - Options
|
|
7158
|
+
* @returns {Promise<Array>} Weekly analytics for the month
|
|
7159
|
+
*/
|
|
7160
|
+
async getMonthByWeek(resourceName, field, month, options = {}) {
|
|
7161
|
+
return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
|
|
7162
|
+
}
|
|
6850
7163
|
/**
|
|
6851
7164
|
* Get top records by volume
|
|
6852
7165
|
* @param {string} resourceName - Resource name
|
|
@@ -6869,6 +7182,8 @@ class FullTextPlugin extends Plugin {
|
|
|
6869
7182
|
...options
|
|
6870
7183
|
};
|
|
6871
7184
|
this.indexes = /* @__PURE__ */ new Map();
|
|
7185
|
+
this.dirtyIndexes = /* @__PURE__ */ new Set();
|
|
7186
|
+
this.deletedIndexes = /* @__PURE__ */ new Set();
|
|
6872
7187
|
}
|
|
6873
7188
|
async onInstall() {
|
|
6874
7189
|
const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
|
|
@@ -6882,7 +7197,11 @@ class FullTextPlugin extends Plugin {
|
|
|
6882
7197
|
// Array of record IDs containing this word
|
|
6883
7198
|
count: "number|required",
|
|
6884
7199
|
lastUpdated: "string|required"
|
|
6885
|
-
}
|
|
7200
|
+
},
|
|
7201
|
+
partitions: {
|
|
7202
|
+
byResource: { fields: { resourceName: "string" } }
|
|
7203
|
+
},
|
|
7204
|
+
behavior: "body-overflow"
|
|
6886
7205
|
}));
|
|
6887
7206
|
this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
|
|
6888
7207
|
await this.loadIndexes();
|
|
@@ -6911,22 +7230,53 @@ class FullTextPlugin extends Plugin {
|
|
|
6911
7230
|
async saveIndexes() {
|
|
6912
7231
|
if (!this.indexResource) return;
|
|
6913
7232
|
const [ok, err] = await tryFn(async () => {
|
|
6914
|
-
const
|
|
6915
|
-
|
|
6916
|
-
await
|
|
7233
|
+
for (const key of this.deletedIndexes) {
|
|
7234
|
+
const [resourceName] = key.split(":");
|
|
7235
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7236
|
+
() => this.indexResource.query({ resourceName })
|
|
7237
|
+
);
|
|
7238
|
+
if (queryOk && results) {
|
|
7239
|
+
for (const index of results) {
|
|
7240
|
+
const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
|
|
7241
|
+
if (indexKey === key) {
|
|
7242
|
+
await this.indexResource.delete(index.id);
|
|
7243
|
+
}
|
|
7244
|
+
}
|
|
7245
|
+
}
|
|
6917
7246
|
}
|
|
6918
|
-
for (const
|
|
7247
|
+
for (const key of this.dirtyIndexes) {
|
|
6919
7248
|
const [resourceName, fieldName, word] = key.split(":");
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
7249
|
+
const data = this.indexes.get(key);
|
|
7250
|
+
if (!data) continue;
|
|
7251
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7252
|
+
() => this.indexResource.query({ resourceName })
|
|
7253
|
+
);
|
|
7254
|
+
let existingRecord = null;
|
|
7255
|
+
if (queryOk && results) {
|
|
7256
|
+
existingRecord = results.find(
|
|
7257
|
+
(index) => index.resourceName === resourceName && index.fieldName === fieldName && index.word === word
|
|
7258
|
+
);
|
|
7259
|
+
}
|
|
7260
|
+
if (existingRecord) {
|
|
7261
|
+
await this.indexResource.update(existingRecord.id, {
|
|
7262
|
+
recordIds: data.recordIds,
|
|
7263
|
+
count: data.count,
|
|
7264
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7265
|
+
});
|
|
7266
|
+
} else {
|
|
7267
|
+
await this.indexResource.insert({
|
|
7268
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
7269
|
+
resourceName,
|
|
7270
|
+
fieldName,
|
|
7271
|
+
word,
|
|
7272
|
+
recordIds: data.recordIds,
|
|
7273
|
+
count: data.count,
|
|
7274
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7275
|
+
});
|
|
7276
|
+
}
|
|
6929
7277
|
}
|
|
7278
|
+
this.dirtyIndexes.clear();
|
|
7279
|
+
this.deletedIndexes.clear();
|
|
6930
7280
|
});
|
|
6931
7281
|
}
|
|
6932
7282
|
installDatabaseHooks() {
|
|
@@ -7021,6 +7371,7 @@ class FullTextPlugin extends Plugin {
|
|
|
7021
7371
|
existing.count = existing.recordIds.length;
|
|
7022
7372
|
}
|
|
7023
7373
|
this.indexes.set(key, existing);
|
|
7374
|
+
this.dirtyIndexes.add(key);
|
|
7024
7375
|
}
|
|
7025
7376
|
}
|
|
7026
7377
|
}
|
|
@@ -7033,8 +7384,10 @@ class FullTextPlugin extends Plugin {
|
|
|
7033
7384
|
data.count = data.recordIds.length;
|
|
7034
7385
|
if (data.recordIds.length === 0) {
|
|
7035
7386
|
this.indexes.delete(key);
|
|
7387
|
+
this.deletedIndexes.add(key);
|
|
7036
7388
|
} else {
|
|
7037
7389
|
this.indexes.set(key, data);
|
|
7390
|
+
this.dirtyIndexes.add(key);
|
|
7038
7391
|
}
|
|
7039
7392
|
}
|
|
7040
7393
|
}
|
|
@@ -7266,8 +7619,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7266
7619
|
errors: "number|required",
|
|
7267
7620
|
avgTime: "number|required",
|
|
7268
7621
|
timestamp: "string|required",
|
|
7269
|
-
metadata: "json"
|
|
7270
|
-
|
|
7622
|
+
metadata: "json",
|
|
7623
|
+
createdAt: "string|required"
|
|
7624
|
+
// YYYY-MM-DD for partitioning
|
|
7625
|
+
},
|
|
7626
|
+
partitions: {
|
|
7627
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7628
|
+
},
|
|
7629
|
+
behavior: "body-overflow"
|
|
7271
7630
|
}));
|
|
7272
7631
|
this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
|
|
7273
7632
|
const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7278,8 +7637,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7278
7637
|
operation: "string|required",
|
|
7279
7638
|
error: "string|required",
|
|
7280
7639
|
timestamp: "string|required",
|
|
7281
|
-
metadata: "json"
|
|
7282
|
-
|
|
7640
|
+
metadata: "json",
|
|
7641
|
+
createdAt: "string|required"
|
|
7642
|
+
// YYYY-MM-DD for partitioning
|
|
7643
|
+
},
|
|
7644
|
+
partitions: {
|
|
7645
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7646
|
+
},
|
|
7647
|
+
behavior: "body-overflow"
|
|
7283
7648
|
}));
|
|
7284
7649
|
this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
|
|
7285
7650
|
const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7290,8 +7655,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7290
7655
|
operation: "string|required",
|
|
7291
7656
|
duration: "number|required",
|
|
7292
7657
|
timestamp: "string|required",
|
|
7293
|
-
metadata: "json"
|
|
7294
|
-
|
|
7658
|
+
metadata: "json",
|
|
7659
|
+
createdAt: "string|required"
|
|
7660
|
+
// YYYY-MM-DD for partitioning
|
|
7661
|
+
},
|
|
7662
|
+
partitions: {
|
|
7663
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7664
|
+
},
|
|
7665
|
+
behavior: "body-overflow"
|
|
7295
7666
|
}));
|
|
7296
7667
|
this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
|
|
7297
7668
|
});
|
|
@@ -7512,6 +7883,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7512
7883
|
errorMetadata = { error: "true" };
|
|
7513
7884
|
resourceMetadata = { resource: "true" };
|
|
7514
7885
|
}
|
|
7886
|
+
const now = /* @__PURE__ */ new Date();
|
|
7887
|
+
const createdAt = now.toISOString().slice(0, 10);
|
|
7515
7888
|
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
7516
7889
|
if (data.count > 0) {
|
|
7517
7890
|
await this.metricsResource.insert({
|
|
@@ -7523,7 +7896,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7523
7896
|
totalTime: data.totalTime,
|
|
7524
7897
|
errors: data.errors,
|
|
7525
7898
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7526
|
-
timestamp:
|
|
7899
|
+
timestamp: now.toISOString(),
|
|
7900
|
+
createdAt,
|
|
7527
7901
|
metadata
|
|
7528
7902
|
});
|
|
7529
7903
|
}
|
|
@@ -7540,7 +7914,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7540
7914
|
totalTime: data.totalTime,
|
|
7541
7915
|
errors: data.errors,
|
|
7542
7916
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7543
|
-
timestamp:
|
|
7917
|
+
timestamp: now.toISOString(),
|
|
7918
|
+
createdAt,
|
|
7544
7919
|
metadata: resourceMetadata
|
|
7545
7920
|
});
|
|
7546
7921
|
}
|
|
@@ -7554,6 +7929,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7554
7929
|
operation: perf.operation,
|
|
7555
7930
|
duration: perf.duration,
|
|
7556
7931
|
timestamp: perf.timestamp,
|
|
7932
|
+
createdAt: perf.timestamp.slice(0, 10),
|
|
7933
|
+
// YYYY-MM-DD from timestamp
|
|
7557
7934
|
metadata: perfMetadata
|
|
7558
7935
|
});
|
|
7559
7936
|
}
|
|
@@ -7567,6 +7944,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7567
7944
|
error: error.error,
|
|
7568
7945
|
stack: error.stack,
|
|
7569
7946
|
timestamp: error.timestamp,
|
|
7947
|
+
createdAt: error.timestamp.slice(0, 10),
|
|
7948
|
+
// YYYY-MM-DD from timestamp
|
|
7570
7949
|
metadata: errorMetadata
|
|
7571
7950
|
});
|
|
7572
7951
|
}
|
|
@@ -7698,22 +8077,47 @@ class MetricsPlugin extends Plugin {
|
|
|
7698
8077
|
async cleanupOldData() {
|
|
7699
8078
|
const cutoffDate = /* @__PURE__ */ new Date();
|
|
7700
8079
|
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
8080
|
+
cutoffDate.toISOString().slice(0, 10);
|
|
8081
|
+
const datesToDelete = [];
|
|
8082
|
+
const startDate = new Date(cutoffDate);
|
|
8083
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
8084
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
8085
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
8086
|
+
}
|
|
7701
8087
|
if (this.metricsResource) {
|
|
7702
|
-
const
|
|
7703
|
-
|
|
7704
|
-
|
|
8088
|
+
for (const dateStr of datesToDelete) {
|
|
8089
|
+
const [ok, err, oldMetrics] = await tryFn(
|
|
8090
|
+
() => this.metricsResource.query({ createdAt: dateStr })
|
|
8091
|
+
);
|
|
8092
|
+
if (ok && oldMetrics) {
|
|
8093
|
+
for (const metric of oldMetrics) {
|
|
8094
|
+
await tryFn(() => this.metricsResource.delete(metric.id));
|
|
8095
|
+
}
|
|
8096
|
+
}
|
|
7705
8097
|
}
|
|
7706
8098
|
}
|
|
7707
8099
|
if (this.errorsResource) {
|
|
7708
|
-
const
|
|
7709
|
-
|
|
7710
|
-
|
|
8100
|
+
for (const dateStr of datesToDelete) {
|
|
8101
|
+
const [ok, err, oldErrors] = await tryFn(
|
|
8102
|
+
() => this.errorsResource.query({ createdAt: dateStr })
|
|
8103
|
+
);
|
|
8104
|
+
if (ok && oldErrors) {
|
|
8105
|
+
for (const error of oldErrors) {
|
|
8106
|
+
await tryFn(() => this.errorsResource.delete(error.id));
|
|
8107
|
+
}
|
|
8108
|
+
}
|
|
7711
8109
|
}
|
|
7712
8110
|
}
|
|
7713
8111
|
if (this.performanceResource) {
|
|
7714
|
-
const
|
|
7715
|
-
|
|
7716
|
-
|
|
8112
|
+
for (const dateStr of datesToDelete) {
|
|
8113
|
+
const [ok, err, oldPerformance] = await tryFn(
|
|
8114
|
+
() => this.performanceResource.query({ createdAt: dateStr })
|
|
8115
|
+
);
|
|
8116
|
+
if (ok && oldPerformance) {
|
|
8117
|
+
for (const perf of oldPerformance) {
|
|
8118
|
+
await tryFn(() => this.performanceResource.delete(perf.id));
|
|
8119
|
+
}
|
|
8120
|
+
}
|
|
7717
8121
|
}
|
|
7718
8122
|
}
|
|
7719
8123
|
}
|
|
@@ -12784,7 +13188,7 @@ class Database extends EventEmitter {
|
|
|
12784
13188
|
this.id = idGenerator(7);
|
|
12785
13189
|
this.version = "1";
|
|
12786
13190
|
this.s3dbVersion = (() => {
|
|
12787
|
-
const [ok, err, version] = tryFn(() => true ? "11.0.
|
|
13191
|
+
const [ok, err, version] = tryFn(() => true ? "11.0.3" : "latest");
|
|
12788
13192
|
return ok ? version : "latest";
|
|
12789
13193
|
})();
|
|
12790
13194
|
this.resources = {};
|
|
@@ -15093,28 +15497,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15093
15497
|
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
15094
15498
|
}
|
|
15095
15499
|
this.queueResource = this.database.resources[queueName];
|
|
15096
|
-
const lockName = `${this.config.resource}_locks`;
|
|
15097
|
-
const [okLock, errLock] = await tryFn(
|
|
15098
|
-
() => this.database.createResource({
|
|
15099
|
-
name: lockName,
|
|
15100
|
-
attributes: {
|
|
15101
|
-
id: "string|required",
|
|
15102
|
-
workerId: "string|required",
|
|
15103
|
-
timestamp: "number|required",
|
|
15104
|
-
ttl: "number|default:5000"
|
|
15105
|
-
},
|
|
15106
|
-
behavior: "body-overflow",
|
|
15107
|
-
timestamps: false
|
|
15108
|
-
})
|
|
15109
|
-
);
|
|
15110
|
-
if (okLock || this.database.resources[lockName]) {
|
|
15111
|
-
this.lockResource = this.database.resources[lockName];
|
|
15112
|
-
} else {
|
|
15113
|
-
this.lockResource = null;
|
|
15114
|
-
if (this.config.verbose) {
|
|
15115
|
-
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
15116
|
-
}
|
|
15117
|
-
}
|
|
15118
15500
|
this.addHelperMethods();
|
|
15119
15501
|
if (this.config.deadLetterResource) {
|
|
15120
15502
|
await this.createDeadLetterResource();
|
|
@@ -15185,13 +15567,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15185
15567
|
}
|
|
15186
15568
|
}
|
|
15187
15569
|
}, 5e3);
|
|
15188
|
-
this.lockCleanupInterval = setInterval(() => {
|
|
15189
|
-
this.cleanupStaleLocks().catch((err) => {
|
|
15190
|
-
if (this.config.verbose) {
|
|
15191
|
-
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
15192
|
-
}
|
|
15193
|
-
});
|
|
15194
|
-
}, 1e4);
|
|
15195
15570
|
for (let i = 0; i < concurrency; i++) {
|
|
15196
15571
|
const worker = this.createWorker(messageHandler, i);
|
|
15197
15572
|
this.workers.push(worker);
|
|
@@ -15208,10 +15583,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15208
15583
|
clearInterval(this.cacheCleanupInterval);
|
|
15209
15584
|
this.cacheCleanupInterval = null;
|
|
15210
15585
|
}
|
|
15211
|
-
if (this.lockCleanupInterval) {
|
|
15212
|
-
clearInterval(this.lockCleanupInterval);
|
|
15213
|
-
this.lockCleanupInterval = null;
|
|
15214
|
-
}
|
|
15215
15586
|
await Promise.all(this.workers);
|
|
15216
15587
|
this.workers = [];
|
|
15217
15588
|
this.processedCache.clear();
|
|
@@ -15262,48 +15633,21 @@ class S3QueuePlugin extends Plugin {
|
|
|
15262
15633
|
return null;
|
|
15263
15634
|
}
|
|
15264
15635
|
/**
|
|
15265
|
-
* Acquire a distributed lock using
|
|
15636
|
+
* Acquire a distributed lock using PluginStorage TTL
|
|
15266
15637
|
* This ensures only one worker can claim a message at a time
|
|
15267
|
-
*
|
|
15268
|
-
* Uses a two-step process:
|
|
15269
|
-
* 1. Create lock resource (similar to queue resource) if not exists
|
|
15270
|
-
* 2. Try to claim lock using ETag-based conditional update
|
|
15271
15638
|
*/
|
|
15272
15639
|
async acquireLock(messageId) {
|
|
15273
|
-
|
|
15274
|
-
|
|
15275
|
-
}
|
|
15276
|
-
const lockId = `lock-${messageId}`;
|
|
15277
|
-
const now = Date.now();
|
|
15640
|
+
const storage = this.getStorage();
|
|
15641
|
+
const lockKey = `msg-${messageId}`;
|
|
15278
15642
|
try {
|
|
15279
|
-
const
|
|
15280
|
-
|
|
15281
|
-
|
|
15282
|
-
|
|
15283
|
-
|
|
15284
|
-
|
|
15285
|
-
|
|
15286
|
-
|
|
15287
|
-
const [ok, err, result] = await tryFn(
|
|
15288
|
-
() => this.lockResource.updateConditional(lockId, {
|
|
15289
|
-
workerId: this.workerId,
|
|
15290
|
-
timestamp: now,
|
|
15291
|
-
ttl: 5e3
|
|
15292
|
-
}, {
|
|
15293
|
-
ifMatch: existingLock._etag
|
|
15294
|
-
})
|
|
15295
|
-
);
|
|
15296
|
-
return ok && result.success;
|
|
15297
|
-
}
|
|
15298
|
-
const [okCreate, errCreate] = await tryFn(
|
|
15299
|
-
() => this.lockResource.insert({
|
|
15300
|
-
id: lockId,
|
|
15301
|
-
workerId: this.workerId,
|
|
15302
|
-
timestamp: now,
|
|
15303
|
-
ttl: 5e3
|
|
15304
|
-
})
|
|
15305
|
-
);
|
|
15306
|
-
return okCreate;
|
|
15643
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
15644
|
+
ttl: 5,
|
|
15645
|
+
// 5 seconds
|
|
15646
|
+
timeout: 0,
|
|
15647
|
+
// Don't wait if locked
|
|
15648
|
+
workerId: this.workerId
|
|
15649
|
+
});
|
|
15650
|
+
return lock !== null;
|
|
15307
15651
|
} catch (error) {
|
|
15308
15652
|
if (this.config.verbose) {
|
|
15309
15653
|
console.log(`[acquireLock] Error: ${error.message}`);
|
|
@@ -15312,15 +15656,13 @@ class S3QueuePlugin extends Plugin {
|
|
|
15312
15656
|
}
|
|
15313
15657
|
}
|
|
15314
15658
|
/**
|
|
15315
|
-
* Release a distributed lock
|
|
15659
|
+
* Release a distributed lock via PluginStorage
|
|
15316
15660
|
*/
|
|
15317
15661
|
async releaseLock(messageId) {
|
|
15318
|
-
|
|
15319
|
-
|
|
15320
|
-
}
|
|
15321
|
-
const lockId = `lock-${messageId}`;
|
|
15662
|
+
const storage = this.getStorage();
|
|
15663
|
+
const lockKey = `msg-${messageId}`;
|
|
15322
15664
|
try {
|
|
15323
|
-
await
|
|
15665
|
+
await storage.releaseLock(lockKey);
|
|
15324
15666
|
} catch (error) {
|
|
15325
15667
|
if (this.config.verbose) {
|
|
15326
15668
|
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
@@ -15328,30 +15670,11 @@ class S3QueuePlugin extends Plugin {
|
|
|
15328
15670
|
}
|
|
15329
15671
|
}
|
|
15330
15672
|
/**
|
|
15331
|
-
* Clean up stale locks
|
|
15332
|
-
*
|
|
15673
|
+
* Clean up stale locks - NO LONGER NEEDED
|
|
15674
|
+
* TTL handles automatic expiration, no manual cleanup required
|
|
15333
15675
|
*/
|
|
15334
15676
|
async cleanupStaleLocks() {
|
|
15335
|
-
|
|
15336
|
-
return;
|
|
15337
|
-
}
|
|
15338
|
-
const now = Date.now();
|
|
15339
|
-
try {
|
|
15340
|
-
const locks = await this.lockResource.list();
|
|
15341
|
-
for (const lock of locks) {
|
|
15342
|
-
const lockAge = now - lock.timestamp;
|
|
15343
|
-
if (lockAge > lock.ttl) {
|
|
15344
|
-
await this.lockResource.delete(lock.id);
|
|
15345
|
-
if (this.config.verbose) {
|
|
15346
|
-
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
15347
|
-
}
|
|
15348
|
-
}
|
|
15349
|
-
}
|
|
15350
|
-
} catch (error) {
|
|
15351
|
-
if (this.config.verbose) {
|
|
15352
|
-
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
15353
|
-
}
|
|
15354
|
-
}
|
|
15677
|
+
return;
|
|
15355
15678
|
}
|
|
15356
15679
|
async attemptClaim(msg) {
|
|
15357
15680
|
const now = Date.now();
|
|
@@ -15575,7 +15898,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15575
15898
|
...options
|
|
15576
15899
|
};
|
|
15577
15900
|
this.database = null;
|
|
15578
|
-
this.lockResource = null;
|
|
15579
15901
|
this.jobs = /* @__PURE__ */ new Map();
|
|
15580
15902
|
this.activeJobs = /* @__PURE__ */ new Map();
|
|
15581
15903
|
this.timers = /* @__PURE__ */ new Map();
|
|
@@ -15614,7 +15936,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15614
15936
|
return true;
|
|
15615
15937
|
}
|
|
15616
15938
|
async onInstall() {
|
|
15617
|
-
await this._createLockResource();
|
|
15618
15939
|
if (this.config.persistJobs) {
|
|
15619
15940
|
await this._createJobHistoryResource();
|
|
15620
15941
|
}
|
|
@@ -15643,25 +15964,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15643
15964
|
await this._startScheduling();
|
|
15644
15965
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
15645
15966
|
}
|
|
15646
|
-
async _createLockResource() {
|
|
15647
|
-
const [ok, err, lockResource] = await tryFn(
|
|
15648
|
-
() => this.database.createResource({
|
|
15649
|
-
name: "plg_scheduler_job_locks",
|
|
15650
|
-
attributes: {
|
|
15651
|
-
id: "string|required",
|
|
15652
|
-
jobName: "string|required",
|
|
15653
|
-
lockedAt: "number|required",
|
|
15654
|
-
instanceId: "string|optional"
|
|
15655
|
-
},
|
|
15656
|
-
behavior: "body-only",
|
|
15657
|
-
timestamps: false
|
|
15658
|
-
})
|
|
15659
|
-
);
|
|
15660
|
-
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
15661
|
-
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
15662
|
-
}
|
|
15663
|
-
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
15664
|
-
}
|
|
15665
15967
|
async _createJobHistoryResource() {
|
|
15666
15968
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
15667
15969
|
name: this.config.jobHistoryResource,
|
|
@@ -15769,16 +16071,16 @@ class SchedulerPlugin extends Plugin {
|
|
|
15769
16071
|
return;
|
|
15770
16072
|
}
|
|
15771
16073
|
this.activeJobs.set(jobName, "acquiring-lock");
|
|
15772
|
-
const
|
|
15773
|
-
const
|
|
15774
|
-
|
|
15775
|
-
|
|
15776
|
-
|
|
15777
|
-
|
|
15778
|
-
|
|
15779
|
-
|
|
15780
|
-
);
|
|
15781
|
-
if (!
|
|
16074
|
+
const storage = this.getStorage();
|
|
16075
|
+
const lockKey = `job-${jobName}`;
|
|
16076
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
16077
|
+
ttl: Math.ceil(job.timeout / 1e3) + 60,
|
|
16078
|
+
// Job timeout + 60 seconds buffer
|
|
16079
|
+
timeout: 0,
|
|
16080
|
+
// Don't wait if locked
|
|
16081
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
16082
|
+
});
|
|
16083
|
+
if (!lock) {
|
|
15782
16084
|
if (this.config.verbose) {
|
|
15783
16085
|
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
15784
16086
|
}
|
|
@@ -15882,7 +16184,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
15882
16184
|
throw lastError;
|
|
15883
16185
|
}
|
|
15884
16186
|
} finally {
|
|
15885
|
-
await tryFn(() =>
|
|
16187
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
15886
16188
|
}
|
|
15887
16189
|
}
|
|
15888
16190
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|