s3db.js 12.2.1 → 12.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1152 -24
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1151 -25
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/plugins/costs.plugin.js +38 -32
- package/src/plugins/geo.plugin.js +1 -1
- package/src/plugins/relation.plugin.js +1 -2
- package/src/plugins/ttl.plugin.js +1 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -6999,18 +6999,13 @@ class CachePlugin extends Plugin {
|
|
|
6999
6999
|
}
|
|
7000
7000
|
}
|
|
7001
7001
|
|
|
7002
|
-
|
|
7003
|
-
|
|
7004
|
-
|
|
7005
|
-
|
|
7006
|
-
|
|
7007
|
-
|
|
7008
|
-
|
|
7009
|
-
considerFreeTier: false,
|
|
7010
|
-
// Flag to consider AWS free tier in calculations
|
|
7011
|
-
region: "us-east-1",
|
|
7012
|
-
// AWS region for pricing (future use)
|
|
7013
|
-
...options
|
|
7002
|
+
class CostsPlugin extends Plugin {
|
|
7003
|
+
constructor(config = {}) {
|
|
7004
|
+
super(config);
|
|
7005
|
+
this.config = {
|
|
7006
|
+
considerFreeTier: config.considerFreeTier !== void 0 ? config.considerFreeTier : false,
|
|
7007
|
+
region: config.region || "us-east-1",
|
|
7008
|
+
...config
|
|
7014
7009
|
};
|
|
7015
7010
|
this.map = {
|
|
7016
7011
|
PutObjectCommand: "put",
|
|
@@ -7104,14 +7099,20 @@ const CostsPlugin = {
|
|
|
7104
7099
|
// Data transfer out cost
|
|
7105
7100
|
}
|
|
7106
7101
|
};
|
|
7102
|
+
}
|
|
7103
|
+
async onInstall() {
|
|
7104
|
+
if (!this.database || !this.database.client) {
|
|
7105
|
+
return;
|
|
7106
|
+
}
|
|
7107
|
+
this.client = this.database.client;
|
|
7107
7108
|
this.client.costs = JSON.parse(JSON.stringify(this.costs));
|
|
7108
|
-
}
|
|
7109
|
-
async
|
|
7109
|
+
}
|
|
7110
|
+
async onStart() {
|
|
7110
7111
|
if (this.client) {
|
|
7111
7112
|
this.client.on("command.response", (name, response, input) => this.addRequest(name, this.map[name], response, input));
|
|
7112
7113
|
this.client.on("command.error", (name, response, input) => this.addRequest(name, this.map[name], response, input));
|
|
7113
7114
|
}
|
|
7114
|
-
}
|
|
7115
|
+
}
|
|
7115
7116
|
addRequest(name, method, response = {}, input = {}) {
|
|
7116
7117
|
if (!method) return;
|
|
7117
7118
|
this.costs.requests.totalEvents++;
|
|
@@ -7151,7 +7152,7 @@ const CostsPlugin = {
|
|
|
7151
7152
|
this.client.costs.requests.subtotal += requestCost;
|
|
7152
7153
|
}
|
|
7153
7154
|
this.updateTotal();
|
|
7154
|
-
}
|
|
7155
|
+
}
|
|
7155
7156
|
trackStorage(bytes) {
|
|
7156
7157
|
this.costs.storage.totalBytes += bytes;
|
|
7157
7158
|
this.costs.storage.totalGB = this.costs.storage.totalBytes / (1024 * 1024 * 1024);
|
|
@@ -7162,7 +7163,7 @@ const CostsPlugin = {
|
|
|
7162
7163
|
this.client.costs.storage.subtotal = this.calculateStorageCost(this.client.costs.storage);
|
|
7163
7164
|
}
|
|
7164
7165
|
this.updateTotal();
|
|
7165
|
-
}
|
|
7166
|
+
}
|
|
7166
7167
|
trackDataTransferIn(bytes) {
|
|
7167
7168
|
this.costs.dataTransfer.inBytes += bytes;
|
|
7168
7169
|
this.costs.dataTransfer.inGB = this.costs.dataTransfer.inBytes / (1024 * 1024 * 1024);
|
|
@@ -7171,7 +7172,7 @@ const CostsPlugin = {
|
|
|
7171
7172
|
this.client.costs.dataTransfer.inGB = this.client.costs.dataTransfer.inBytes / (1024 * 1024 * 1024);
|
|
7172
7173
|
}
|
|
7173
7174
|
this.updateTotal();
|
|
7174
|
-
}
|
|
7175
|
+
}
|
|
7175
7176
|
trackDataTransferOut(bytes) {
|
|
7176
7177
|
this.costs.dataTransfer.outBytes += bytes;
|
|
7177
7178
|
this.costs.dataTransfer.outGB = this.costs.dataTransfer.outBytes / (1024 * 1024 * 1024);
|
|
@@ -7182,7 +7183,7 @@ const CostsPlugin = {
|
|
|
7182
7183
|
this.client.costs.dataTransfer.subtotal = this.calculateDataTransferCost(this.client.costs.dataTransfer);
|
|
7183
7184
|
}
|
|
7184
7185
|
this.updateTotal();
|
|
7185
|
-
}
|
|
7186
|
+
}
|
|
7186
7187
|
calculateStorageCost(storage) {
|
|
7187
7188
|
const totalGB = storage.totalGB;
|
|
7188
7189
|
let cost = 0;
|
|
@@ -7201,11 +7202,11 @@ const CostsPlugin = {
|
|
|
7201
7202
|
}
|
|
7202
7203
|
}
|
|
7203
7204
|
return cost;
|
|
7204
|
-
}
|
|
7205
|
+
}
|
|
7205
7206
|
calculateDataTransferCost(dataTransfer) {
|
|
7206
7207
|
let totalGB = dataTransfer.outGB;
|
|
7207
7208
|
let cost = 0;
|
|
7208
|
-
if (this.
|
|
7209
|
+
if (this.config && this.config.considerFreeTier) {
|
|
7209
7210
|
const freeTierRemaining = dataTransfer.freeTierGB - dataTransfer.freeTierUsed;
|
|
7210
7211
|
if (freeTierRemaining > 0 && totalGB > 0) {
|
|
7211
7212
|
const gbToDeduct = Math.min(totalGB, freeTierRemaining);
|
|
@@ -7228,14 +7229,14 @@ const CostsPlugin = {
|
|
|
7228
7229
|
}
|
|
7229
7230
|
}
|
|
7230
7231
|
return cost;
|
|
7231
|
-
}
|
|
7232
|
+
}
|
|
7232
7233
|
updateTotal() {
|
|
7233
7234
|
this.costs.total = this.costs.requests.subtotal + this.costs.storage.subtotal + this.costs.dataTransfer.subtotal;
|
|
7234
7235
|
if (this.client && this.client.costs) {
|
|
7235
7236
|
this.client.costs.total = this.client.costs.requests.subtotal + this.client.costs.storage.subtotal + this.client.costs.dataTransfer.subtotal;
|
|
7236
7237
|
}
|
|
7237
7238
|
}
|
|
7238
|
-
}
|
|
7239
|
+
}
|
|
7239
7240
|
|
|
7240
7241
|
function createConfig(options, detectedTimezone) {
|
|
7241
7242
|
const consolidation = options.consolidation || {};
|
|
@@ -10645,6 +10646,605 @@ class FullTextPlugin extends Plugin {
|
|
|
10645
10646
|
}
|
|
10646
10647
|
}
|
|
10647
10648
|
|
|
10649
|
+
class GeoPlugin extends Plugin {
|
|
10650
|
+
constructor(config = {}) {
|
|
10651
|
+
super(config);
|
|
10652
|
+
this.resources = config.resources || {};
|
|
10653
|
+
this.verbose = config.verbose !== void 0 ? config.verbose : false;
|
|
10654
|
+
this.base32 = "0123456789bcdefghjkmnpqrstuvwxyz";
|
|
10655
|
+
}
|
|
10656
|
+
/**
|
|
10657
|
+
* Install the plugin
|
|
10658
|
+
*/
|
|
10659
|
+
async install(database) {
|
|
10660
|
+
await super.install(database);
|
|
10661
|
+
for (const [resourceName, config] of Object.entries(this.resources)) {
|
|
10662
|
+
await this._setupResource(resourceName, config);
|
|
10663
|
+
}
|
|
10664
|
+
this.database.addHook("afterCreateResource", async (context) => {
|
|
10665
|
+
const { resource, config: resourceConfig } = context;
|
|
10666
|
+
const geoConfig = this.resources[resource.name];
|
|
10667
|
+
if (geoConfig) {
|
|
10668
|
+
await this._setupResource(resource.name, geoConfig);
|
|
10669
|
+
}
|
|
10670
|
+
});
|
|
10671
|
+
if (this.verbose) {
|
|
10672
|
+
console.log(`[GeoPlugin] Installed with ${Object.keys(this.resources).length} resources`);
|
|
10673
|
+
}
|
|
10674
|
+
this.emit("installed", {
|
|
10675
|
+
plugin: "GeoPlugin",
|
|
10676
|
+
resources: Object.keys(this.resources)
|
|
10677
|
+
});
|
|
10678
|
+
}
|
|
10679
|
+
/**
|
|
10680
|
+
* Setup a resource with geo capabilities
|
|
10681
|
+
*/
|
|
10682
|
+
async _setupResource(resourceName, config) {
|
|
10683
|
+
if (!this.database.resources[resourceName]) {
|
|
10684
|
+
if (this.verbose) {
|
|
10685
|
+
console.warn(`[GeoPlugin] Resource "${resourceName}" not found, will setup when created`);
|
|
10686
|
+
}
|
|
10687
|
+
return;
|
|
10688
|
+
}
|
|
10689
|
+
const resource = this.database.resources[resourceName];
|
|
10690
|
+
if (!resource || typeof resource.addHook !== "function") {
|
|
10691
|
+
if (this.verbose) {
|
|
10692
|
+
console.warn(`[GeoPlugin] Resource "${resourceName}" not found or invalid`);
|
|
10693
|
+
}
|
|
10694
|
+
return;
|
|
10695
|
+
}
|
|
10696
|
+
if (!config.latField || !config.lonField) {
|
|
10697
|
+
throw new Error(
|
|
10698
|
+
`[GeoPlugin] Resource "${resourceName}" must have "latField" and "lonField" configured`
|
|
10699
|
+
);
|
|
10700
|
+
}
|
|
10701
|
+
if (!config.precision || config.precision < 1 || config.precision > 12) {
|
|
10702
|
+
config.precision = 5;
|
|
10703
|
+
}
|
|
10704
|
+
resource._geoConfig = config;
|
|
10705
|
+
const latField = resource.attributes[config.latField];
|
|
10706
|
+
const lonField = resource.attributes[config.lonField];
|
|
10707
|
+
const isLatOptional = typeof latField === "object" && latField.optional === true;
|
|
10708
|
+
const isLonOptional = typeof lonField === "object" && lonField.optional === true;
|
|
10709
|
+
const areCoordinatesOptional = isLatOptional || isLonOptional;
|
|
10710
|
+
const geohashType = areCoordinatesOptional ? "string|optional" : "string";
|
|
10711
|
+
let needsUpdate = false;
|
|
10712
|
+
const newAttributes = { ...resource.attributes };
|
|
10713
|
+
if (config.addGeohash && !newAttributes.geohash) {
|
|
10714
|
+
newAttributes.geohash = geohashType;
|
|
10715
|
+
needsUpdate = true;
|
|
10716
|
+
}
|
|
10717
|
+
if (!newAttributes._geohash) {
|
|
10718
|
+
newAttributes._geohash = geohashType;
|
|
10719
|
+
needsUpdate = true;
|
|
10720
|
+
}
|
|
10721
|
+
if (config.zoomLevels && Array.isArray(config.zoomLevels)) {
|
|
10722
|
+
for (const zoom of config.zoomLevels) {
|
|
10723
|
+
const fieldName = `_geohash_zoom${zoom}`;
|
|
10724
|
+
if (!newAttributes[fieldName]) {
|
|
10725
|
+
newAttributes[fieldName] = geohashType;
|
|
10726
|
+
needsUpdate = true;
|
|
10727
|
+
}
|
|
10728
|
+
}
|
|
10729
|
+
}
|
|
10730
|
+
if (needsUpdate) {
|
|
10731
|
+
resource.updateAttributes(newAttributes);
|
|
10732
|
+
if (this.database.uploadMetadataFile) {
|
|
10733
|
+
await this.database.uploadMetadataFile();
|
|
10734
|
+
}
|
|
10735
|
+
}
|
|
10736
|
+
if (config.usePartitions) {
|
|
10737
|
+
await this._setupPartitions(resource, config);
|
|
10738
|
+
}
|
|
10739
|
+
this._addHooks(resource, config);
|
|
10740
|
+
this._addHelperMethods(resource, config);
|
|
10741
|
+
if (this.verbose) {
|
|
10742
|
+
console.log(
|
|
10743
|
+
`[GeoPlugin] Setup resource "${resourceName}" with precision ${config.precision} (~${this._getPrecisionDistance(config.precision)}km cells)` + (config.usePartitions ? " [Partitions enabled]" : "")
|
|
10744
|
+
);
|
|
10745
|
+
}
|
|
10746
|
+
}
|
|
10747
|
+
/**
|
|
10748
|
+
* Setup geohash partitions for efficient spatial queries
|
|
10749
|
+
* Creates multiple zoom-level partitions if zoomLevels configured
|
|
10750
|
+
*/
|
|
10751
|
+
async _setupPartitions(resource, config) {
|
|
10752
|
+
const updatedConfig = { ...resource.config };
|
|
10753
|
+
updatedConfig.partitions = updatedConfig.partitions || {};
|
|
10754
|
+
let partitionsCreated = 0;
|
|
10755
|
+
if (config.zoomLevels && Array.isArray(config.zoomLevels)) {
|
|
10756
|
+
for (const zoom of config.zoomLevels) {
|
|
10757
|
+
const partitionName = `byGeohashZoom${zoom}`;
|
|
10758
|
+
const fieldName = `_geohash_zoom${zoom}`;
|
|
10759
|
+
if (!updatedConfig.partitions[partitionName]) {
|
|
10760
|
+
updatedConfig.partitions[partitionName] = {
|
|
10761
|
+
fields: {
|
|
10762
|
+
[fieldName]: "string"
|
|
10763
|
+
}
|
|
10764
|
+
};
|
|
10765
|
+
partitionsCreated++;
|
|
10766
|
+
if (this.verbose) {
|
|
10767
|
+
console.log(
|
|
10768
|
+
`[GeoPlugin] Created ${partitionName} partition for "${resource.name}" (precision ${zoom}, ~${this._getPrecisionDistance(zoom)}km cells)`
|
|
10769
|
+
);
|
|
10770
|
+
}
|
|
10771
|
+
}
|
|
10772
|
+
}
|
|
10773
|
+
} else {
|
|
10774
|
+
const hasGeohashPartition = resource.config.partitions && resource.config.partitions.byGeohash;
|
|
10775
|
+
if (!hasGeohashPartition) {
|
|
10776
|
+
updatedConfig.partitions.byGeohash = {
|
|
10777
|
+
fields: {
|
|
10778
|
+
_geohash: "string"
|
|
10779
|
+
}
|
|
10780
|
+
};
|
|
10781
|
+
partitionsCreated++;
|
|
10782
|
+
if (this.verbose) {
|
|
10783
|
+
console.log(`[GeoPlugin] Created byGeohash partition for "${resource.name}"`);
|
|
10784
|
+
}
|
|
10785
|
+
}
|
|
10786
|
+
}
|
|
10787
|
+
if (partitionsCreated > 0) {
|
|
10788
|
+
resource.config = updatedConfig;
|
|
10789
|
+
resource.setupPartitionHooks();
|
|
10790
|
+
if (this.database.uploadMetadataFile) {
|
|
10791
|
+
await this.database.uploadMetadataFile();
|
|
10792
|
+
}
|
|
10793
|
+
}
|
|
10794
|
+
}
|
|
10795
|
+
/**
|
|
10796
|
+
* Add hooks to automatically calculate geohash at all zoom levels
|
|
10797
|
+
*/
|
|
10798
|
+
_addHooks(resource, config) {
|
|
10799
|
+
const calculateGeohash = async (data) => {
|
|
10800
|
+
const lat = data[config.latField];
|
|
10801
|
+
const lon = data[config.lonField];
|
|
10802
|
+
if (lat !== void 0 && lon !== void 0) {
|
|
10803
|
+
const geohash = this.encodeGeohash(lat, lon, config.precision);
|
|
10804
|
+
if (config.addGeohash) {
|
|
10805
|
+
data.geohash = geohash;
|
|
10806
|
+
}
|
|
10807
|
+
data._geohash = geohash;
|
|
10808
|
+
if (config.zoomLevels && Array.isArray(config.zoomLevels)) {
|
|
10809
|
+
for (const zoom of config.zoomLevels) {
|
|
10810
|
+
const zoomGeohash = this.encodeGeohash(lat, lon, zoom);
|
|
10811
|
+
data[`_geohash_zoom${zoom}`] = zoomGeohash;
|
|
10812
|
+
}
|
|
10813
|
+
}
|
|
10814
|
+
}
|
|
10815
|
+
return data;
|
|
10816
|
+
};
|
|
10817
|
+
resource.addHook("beforeInsert", calculateGeohash);
|
|
10818
|
+
resource.addHook("beforeUpdate", calculateGeohash);
|
|
10819
|
+
}
|
|
10820
|
+
/**
|
|
10821
|
+
* Add helper methods to resource
|
|
10822
|
+
*/
|
|
10823
|
+
_addHelperMethods(resource, config) {
|
|
10824
|
+
const plugin = this;
|
|
10825
|
+
resource.findNearby = async function({ lat, lon, radius = 10, limit = 100 }) {
|
|
10826
|
+
if (lat === void 0 || lon === void 0) {
|
|
10827
|
+
throw new Error("lat and lon are required for findNearby");
|
|
10828
|
+
}
|
|
10829
|
+
const longitude = lon;
|
|
10830
|
+
let allRecords = [];
|
|
10831
|
+
if (config.usePartitions) {
|
|
10832
|
+
let partitionName, fieldName, precision;
|
|
10833
|
+
if (config.zoomLevels && config.zoomLevels.length > 0) {
|
|
10834
|
+
const optimalZoom = plugin._selectOptimalZoom(config.zoomLevels, radius);
|
|
10835
|
+
partitionName = `byGeohashZoom${optimalZoom}`;
|
|
10836
|
+
fieldName = `_geohash_zoom${optimalZoom}`;
|
|
10837
|
+
precision = optimalZoom;
|
|
10838
|
+
if (plugin.verbose) {
|
|
10839
|
+
console.log(
|
|
10840
|
+
`[GeoPlugin] Auto-selected zoom${optimalZoom} (${plugin._getPrecisionDistance(optimalZoom)}km cells) for ${radius}km radius query`
|
|
10841
|
+
);
|
|
10842
|
+
}
|
|
10843
|
+
} else {
|
|
10844
|
+
partitionName = "byGeohash";
|
|
10845
|
+
fieldName = "_geohash";
|
|
10846
|
+
precision = config.precision;
|
|
10847
|
+
}
|
|
10848
|
+
if (this.config.partitions?.[partitionName]) {
|
|
10849
|
+
const centerGeohash = plugin.encodeGeohash(lat, longitude, precision);
|
|
10850
|
+
const neighbors = plugin.getNeighbors(centerGeohash);
|
|
10851
|
+
const geohashesToSearch = [centerGeohash, ...neighbors];
|
|
10852
|
+
const partitionResults = await Promise.all(
|
|
10853
|
+
geohashesToSearch.map(async (geohash) => {
|
|
10854
|
+
const [ok, err, records] = await tryFn(async () => {
|
|
10855
|
+
return await this.listPartition({
|
|
10856
|
+
partition: partitionName,
|
|
10857
|
+
partitionValues: { [fieldName]: geohash },
|
|
10858
|
+
limit: limit * 2
|
|
10859
|
+
});
|
|
10860
|
+
});
|
|
10861
|
+
return ok ? records : [];
|
|
10862
|
+
})
|
|
10863
|
+
);
|
|
10864
|
+
allRecords = partitionResults.flat();
|
|
10865
|
+
if (plugin.verbose) {
|
|
10866
|
+
console.log(
|
|
10867
|
+
`[GeoPlugin] findNearby searched ${geohashesToSearch.length} ${partitionName} partitions, found ${allRecords.length} candidates`
|
|
10868
|
+
);
|
|
10869
|
+
}
|
|
10870
|
+
} else {
|
|
10871
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
10872
|
+
}
|
|
10873
|
+
} else {
|
|
10874
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
10875
|
+
}
|
|
10876
|
+
const withDistances = allRecords.map((record) => {
|
|
10877
|
+
const recordLat = record[config.latField];
|
|
10878
|
+
const recordLon = record[config.lonField];
|
|
10879
|
+
if (recordLat === void 0 || recordLon === void 0) {
|
|
10880
|
+
return null;
|
|
10881
|
+
}
|
|
10882
|
+
const distance = plugin.calculateDistance(lat, longitude, recordLat, recordLon);
|
|
10883
|
+
return {
|
|
10884
|
+
...record,
|
|
10885
|
+
_distance: distance
|
|
10886
|
+
};
|
|
10887
|
+
}).filter((record) => record !== null && record._distance <= radius).sort((a, b) => a._distance - b._distance).slice(0, limit);
|
|
10888
|
+
return withDistances;
|
|
10889
|
+
};
|
|
10890
|
+
resource.findInBounds = async function({ north, south, east, west, limit = 100 }) {
|
|
10891
|
+
if (north === void 0 || south === void 0 || east === void 0 || west === void 0) {
|
|
10892
|
+
throw new Error("north, south, east, west are required for findInBounds");
|
|
10893
|
+
}
|
|
10894
|
+
let allRecords = [];
|
|
10895
|
+
if (config.usePartitions) {
|
|
10896
|
+
let partitionName, precision;
|
|
10897
|
+
if (config.zoomLevels && config.zoomLevels.length > 0) {
|
|
10898
|
+
const centerLat = (north + south) / 2;
|
|
10899
|
+
const centerLon = (east + west) / 2;
|
|
10900
|
+
const latRadius = plugin.calculateDistance(centerLat, centerLon, north, centerLon);
|
|
10901
|
+
const lonRadius = plugin.calculateDistance(centerLat, centerLon, centerLat, east);
|
|
10902
|
+
const approximateRadius = Math.max(latRadius, lonRadius);
|
|
10903
|
+
const optimalZoom = plugin._selectOptimalZoom(config.zoomLevels, approximateRadius);
|
|
10904
|
+
partitionName = `byGeohashZoom${optimalZoom}`;
|
|
10905
|
+
precision = optimalZoom;
|
|
10906
|
+
if (plugin.verbose) {
|
|
10907
|
+
console.log(
|
|
10908
|
+
`[GeoPlugin] Auto-selected zoom${optimalZoom} (${plugin._getPrecisionDistance(optimalZoom)}km cells) for ${approximateRadius.toFixed(1)}km bounding box`
|
|
10909
|
+
);
|
|
10910
|
+
}
|
|
10911
|
+
} else {
|
|
10912
|
+
partitionName = "byGeohash";
|
|
10913
|
+
precision = config.precision;
|
|
10914
|
+
}
|
|
10915
|
+
if (this.config.partitions?.[partitionName]) {
|
|
10916
|
+
const geohashesToSearch = plugin._getGeohashesInBounds({
|
|
10917
|
+
north,
|
|
10918
|
+
south,
|
|
10919
|
+
east,
|
|
10920
|
+
west,
|
|
10921
|
+
precision
|
|
10922
|
+
});
|
|
10923
|
+
const partitionResults = await Promise.all(
|
|
10924
|
+
geohashesToSearch.map(async (geohash) => {
|
|
10925
|
+
const [ok, err, records] = await tryFn(async () => {
|
|
10926
|
+
const fieldName = config.zoomLevels ? `_geohash_zoom${precision}` : "_geohash";
|
|
10927
|
+
return await this.listPartition({
|
|
10928
|
+
partition: partitionName,
|
|
10929
|
+
partitionValues: { [fieldName]: geohash },
|
|
10930
|
+
limit: limit * 2
|
|
10931
|
+
});
|
|
10932
|
+
});
|
|
10933
|
+
return ok ? records : [];
|
|
10934
|
+
})
|
|
10935
|
+
);
|
|
10936
|
+
allRecords = partitionResults.flat();
|
|
10937
|
+
if (plugin.verbose) {
|
|
10938
|
+
console.log(
|
|
10939
|
+
`[GeoPlugin] findInBounds searched ${geohashesToSearch.length} ${partitionName} partitions, found ${allRecords.length} candidates`
|
|
10940
|
+
);
|
|
10941
|
+
}
|
|
10942
|
+
} else {
|
|
10943
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
10944
|
+
}
|
|
10945
|
+
} else {
|
|
10946
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
10947
|
+
}
|
|
10948
|
+
const inBounds = allRecords.filter((record) => {
|
|
10949
|
+
const lat = record[config.latField];
|
|
10950
|
+
const lon = record[config.lonField];
|
|
10951
|
+
if (lat === void 0 || lon === void 0) {
|
|
10952
|
+
return false;
|
|
10953
|
+
}
|
|
10954
|
+
return lat <= north && lat >= south && lon <= east && lon >= west;
|
|
10955
|
+
}).slice(0, limit);
|
|
10956
|
+
return inBounds;
|
|
10957
|
+
};
|
|
10958
|
+
resource.getDistance = async function(id1, id2) {
|
|
10959
|
+
let record1, record2;
|
|
10960
|
+
try {
|
|
10961
|
+
[record1, record2] = await Promise.all([
|
|
10962
|
+
this.get(id1),
|
|
10963
|
+
this.get(id2)
|
|
10964
|
+
]);
|
|
10965
|
+
} catch (err) {
|
|
10966
|
+
if (err.name === "NoSuchKey" || err.message?.includes("No such key")) {
|
|
10967
|
+
throw new Error("One or both records not found");
|
|
10968
|
+
}
|
|
10969
|
+
throw err;
|
|
10970
|
+
}
|
|
10971
|
+
if (!record1 || !record2) {
|
|
10972
|
+
throw new Error("One or both records not found");
|
|
10973
|
+
}
|
|
10974
|
+
const lat1 = record1[config.latField];
|
|
10975
|
+
const lon1 = record1[config.lonField];
|
|
10976
|
+
const lat2 = record2[config.latField];
|
|
10977
|
+
const lon2 = record2[config.lonField];
|
|
10978
|
+
if (lat1 === void 0 || lon1 === void 0 || lat2 === void 0 || lon2 === void 0) {
|
|
10979
|
+
throw new Error("One or both records missing coordinates");
|
|
10980
|
+
}
|
|
10981
|
+
const distance = plugin.calculateDistance(lat1, lon1, lat2, lon2);
|
|
10982
|
+
return {
|
|
10983
|
+
distance,
|
|
10984
|
+
unit: "km",
|
|
10985
|
+
from: id1,
|
|
10986
|
+
to: id2
|
|
10987
|
+
};
|
|
10988
|
+
};
|
|
10989
|
+
}
|
|
10990
|
+
/**
|
|
10991
|
+
* Encode coordinates to geohash
|
|
10992
|
+
* @param {number} latitude - Latitude (-90 to 90)
|
|
10993
|
+
* @param {number} longitude - Longitude (-180 to 180)
|
|
10994
|
+
* @param {number} precision - Number of characters in geohash
|
|
10995
|
+
* @returns {string} Geohash string
|
|
10996
|
+
*/
|
|
10997
|
+
encodeGeohash(latitude, longitude, precision = 5) {
|
|
10998
|
+
let idx = 0;
|
|
10999
|
+
let bit = 0;
|
|
11000
|
+
let evenBit = true;
|
|
11001
|
+
let geohash = "";
|
|
11002
|
+
let latMin = -90;
|
|
11003
|
+
let latMax = 90;
|
|
11004
|
+
let lonMin = -180;
|
|
11005
|
+
let lonMax = 180;
|
|
11006
|
+
while (geohash.length < precision) {
|
|
11007
|
+
if (evenBit) {
|
|
11008
|
+
const lonMid = (lonMin + lonMax) / 2;
|
|
11009
|
+
if (longitude > lonMid) {
|
|
11010
|
+
idx |= 1 << 4 - bit;
|
|
11011
|
+
lonMin = lonMid;
|
|
11012
|
+
} else {
|
|
11013
|
+
lonMax = lonMid;
|
|
11014
|
+
}
|
|
11015
|
+
} else {
|
|
11016
|
+
const latMid = (latMin + latMax) / 2;
|
|
11017
|
+
if (latitude > latMid) {
|
|
11018
|
+
idx |= 1 << 4 - bit;
|
|
11019
|
+
latMin = latMid;
|
|
11020
|
+
} else {
|
|
11021
|
+
latMax = latMid;
|
|
11022
|
+
}
|
|
11023
|
+
}
|
|
11024
|
+
evenBit = !evenBit;
|
|
11025
|
+
if (bit < 4) {
|
|
11026
|
+
bit++;
|
|
11027
|
+
} else {
|
|
11028
|
+
geohash += this.base32[idx];
|
|
11029
|
+
bit = 0;
|
|
11030
|
+
idx = 0;
|
|
11031
|
+
}
|
|
11032
|
+
}
|
|
11033
|
+
return geohash;
|
|
11034
|
+
}
|
|
11035
|
+
/**
|
|
11036
|
+
* Decode geohash to coordinates
|
|
11037
|
+
* @param {string} geohash - Geohash string
|
|
11038
|
+
* @returns {Object} { latitude, longitude, error }
|
|
11039
|
+
*/
|
|
11040
|
+
decodeGeohash(geohash) {
|
|
11041
|
+
let evenBit = true;
|
|
11042
|
+
let latMin = -90;
|
|
11043
|
+
let latMax = 90;
|
|
11044
|
+
let lonMin = -180;
|
|
11045
|
+
let lonMax = 180;
|
|
11046
|
+
for (let i = 0; i < geohash.length; i++) {
|
|
11047
|
+
const chr = geohash[i];
|
|
11048
|
+
const idx = this.base32.indexOf(chr);
|
|
11049
|
+
if (idx === -1) {
|
|
11050
|
+
throw new Error(`Invalid geohash character: ${chr}`);
|
|
11051
|
+
}
|
|
11052
|
+
for (let n = 4; n >= 0; n--) {
|
|
11053
|
+
const bitN = idx >> n & 1;
|
|
11054
|
+
if (evenBit) {
|
|
11055
|
+
const lonMid = (lonMin + lonMax) / 2;
|
|
11056
|
+
if (bitN === 1) {
|
|
11057
|
+
lonMin = lonMid;
|
|
11058
|
+
} else {
|
|
11059
|
+
lonMax = lonMid;
|
|
11060
|
+
}
|
|
11061
|
+
} else {
|
|
11062
|
+
const latMid = (latMin + latMax) / 2;
|
|
11063
|
+
if (bitN === 1) {
|
|
11064
|
+
latMin = latMid;
|
|
11065
|
+
} else {
|
|
11066
|
+
latMax = latMid;
|
|
11067
|
+
}
|
|
11068
|
+
}
|
|
11069
|
+
evenBit = !evenBit;
|
|
11070
|
+
}
|
|
11071
|
+
}
|
|
11072
|
+
const latitude = (latMin + latMax) / 2;
|
|
11073
|
+
const longitude = (lonMin + lonMax) / 2;
|
|
11074
|
+
return {
|
|
11075
|
+
latitude,
|
|
11076
|
+
longitude,
|
|
11077
|
+
error: {
|
|
11078
|
+
latitude: latMax - latMin,
|
|
11079
|
+
longitude: lonMax - lonMin
|
|
11080
|
+
}
|
|
11081
|
+
};
|
|
11082
|
+
}
|
|
11083
|
+
/**
|
|
11084
|
+
* Calculate distance between two coordinates using Haversine formula
|
|
11085
|
+
* @param {number} lat1 - Latitude of point 1
|
|
11086
|
+
* @param {number} lon1 - Longitude of point 1
|
|
11087
|
+
* @param {number} lat2 - Latitude of point 2
|
|
11088
|
+
* @param {number} lon2 - Longitude of point 2
|
|
11089
|
+
* @returns {number} Distance in kilometers
|
|
11090
|
+
*/
|
|
11091
|
+
calculateDistance(lat1, lon1, lat2, lon2) {
|
|
11092
|
+
const R = 6371;
|
|
11093
|
+
const dLat = this._toRadians(lat2 - lat1);
|
|
11094
|
+
const dLon = this._toRadians(lon2 - lon1);
|
|
11095
|
+
const a = Math.sin(dLat / 2) * Math.sin(dLat / 2) + Math.cos(this._toRadians(lat1)) * Math.cos(this._toRadians(lat2)) * Math.sin(dLon / 2) * Math.sin(dLon / 2);
|
|
11096
|
+
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
|
11097
|
+
return R * c;
|
|
11098
|
+
}
|
|
11099
|
+
/**
|
|
11100
|
+
* Get geohash neighbors (8 surrounding cells)
|
|
11101
|
+
* @param {string} geohash - Center geohash
|
|
11102
|
+
* @returns {Array<string>} Array of 8 neighboring geohashes
|
|
11103
|
+
*/
|
|
11104
|
+
getNeighbors(geohash) {
|
|
11105
|
+
const decoded = this.decodeGeohash(geohash);
|
|
11106
|
+
const { latitude, longitude, error } = decoded;
|
|
11107
|
+
const latStep = error.latitude;
|
|
11108
|
+
const lonStep = error.longitude;
|
|
11109
|
+
const neighbors = [];
|
|
11110
|
+
const directions = [
|
|
11111
|
+
[-latStep, -lonStep],
|
|
11112
|
+
// SW
|
|
11113
|
+
[-latStep, 0],
|
|
11114
|
+
// S
|
|
11115
|
+
[-latStep, lonStep],
|
|
11116
|
+
// SE
|
|
11117
|
+
[0, -lonStep],
|
|
11118
|
+
// W
|
|
11119
|
+
[0, lonStep],
|
|
11120
|
+
// E
|
|
11121
|
+
[latStep, -lonStep],
|
|
11122
|
+
// NW
|
|
11123
|
+
[latStep, 0],
|
|
11124
|
+
// N
|
|
11125
|
+
[latStep, lonStep]
|
|
11126
|
+
// NE
|
|
11127
|
+
];
|
|
11128
|
+
for (const [latDelta, lonDelta] of directions) {
|
|
11129
|
+
const neighborHash = this.encodeGeohash(
|
|
11130
|
+
latitude + latDelta,
|
|
11131
|
+
longitude + lonDelta,
|
|
11132
|
+
geohash.length
|
|
11133
|
+
);
|
|
11134
|
+
neighbors.push(neighborHash);
|
|
11135
|
+
}
|
|
11136
|
+
return neighbors;
|
|
11137
|
+
}
|
|
11138
|
+
/**
|
|
11139
|
+
* Get all geohashes that cover a bounding box
|
|
11140
|
+
* @param {Object} bounds - Bounding box { north, south, east, west, precision }
|
|
11141
|
+
* @returns {Array<string>} Array of unique geohashes covering the area
|
|
11142
|
+
*/
|
|
11143
|
+
_getGeohashesInBounds({ north, south, east, west, precision }) {
|
|
11144
|
+
const geohashes = /* @__PURE__ */ new Set();
|
|
11145
|
+
const cellSize = this._getPrecisionDistance(precision);
|
|
11146
|
+
const latStep = cellSize / 111;
|
|
11147
|
+
const lonStep = cellSize / (111 * Math.cos(this._toRadians((north + south) / 2)));
|
|
11148
|
+
for (let lat = south; lat <= north; lat += latStep) {
|
|
11149
|
+
for (let lon = west; lon <= east; lon += lonStep) {
|
|
11150
|
+
const geohash = this.encodeGeohash(lat, lon, precision);
|
|
11151
|
+
geohashes.add(geohash);
|
|
11152
|
+
}
|
|
11153
|
+
}
|
|
11154
|
+
const corners = [
|
|
11155
|
+
[north, west],
|
|
11156
|
+
[north, east],
|
|
11157
|
+
[south, west],
|
|
11158
|
+
[south, east],
|
|
11159
|
+
[(north + south) / 2, west],
|
|
11160
|
+
[(north + south) / 2, east],
|
|
11161
|
+
[north, (east + west) / 2],
|
|
11162
|
+
[south, (east + west) / 2]
|
|
11163
|
+
];
|
|
11164
|
+
for (const [lat, lon] of corners) {
|
|
11165
|
+
const geohash = this.encodeGeohash(lat, lon, precision);
|
|
11166
|
+
geohashes.add(geohash);
|
|
11167
|
+
}
|
|
11168
|
+
return Array.from(geohashes);
|
|
11169
|
+
}
|
|
11170
|
+
/**
|
|
11171
|
+
* Convert degrees to radians
|
|
11172
|
+
*/
|
|
11173
|
+
_toRadians(degrees) {
|
|
11174
|
+
return degrees * (Math.PI / 180);
|
|
11175
|
+
}
|
|
11176
|
+
/**
|
|
11177
|
+
* Get approximate cell size for precision level
|
|
11178
|
+
*/
|
|
11179
|
+
_getPrecisionDistance(precision) {
|
|
11180
|
+
const distances = {
|
|
11181
|
+
1: 5e3,
|
|
11182
|
+
2: 1250,
|
|
11183
|
+
3: 156,
|
|
11184
|
+
4: 39,
|
|
11185
|
+
5: 4.9,
|
|
11186
|
+
6: 1.2,
|
|
11187
|
+
7: 0.15,
|
|
11188
|
+
8: 0.038,
|
|
11189
|
+
9: 47e-4,
|
|
11190
|
+
10: 12e-4,
|
|
11191
|
+
11: 15e-5,
|
|
11192
|
+
12: 37e-6
|
|
11193
|
+
};
|
|
11194
|
+
return distances[precision] || 5;
|
|
11195
|
+
}
|
|
11196
|
+
/**
|
|
11197
|
+
* Select optimal zoom level based on search radius
|
|
11198
|
+
* @param {Array<number>} zoomLevels - Available zoom levels
|
|
11199
|
+
* @param {number} radiusKm - Search radius in kilometers
|
|
11200
|
+
* @returns {number} Optimal zoom precision
|
|
11201
|
+
*/
|
|
11202
|
+
_selectOptimalZoom(zoomLevels, radiusKm) {
|
|
11203
|
+
if (!zoomLevels || zoomLevels.length === 0) {
|
|
11204
|
+
return null;
|
|
11205
|
+
}
|
|
11206
|
+
const targetCellSize = radiusKm / 2.5;
|
|
11207
|
+
let bestZoom = zoomLevels[0];
|
|
11208
|
+
let bestDiff = Math.abs(this._getPrecisionDistance(bestZoom) - targetCellSize);
|
|
11209
|
+
for (const zoom of zoomLevels) {
|
|
11210
|
+
const cellSize = this._getPrecisionDistance(zoom);
|
|
11211
|
+
const diff = Math.abs(cellSize - targetCellSize);
|
|
11212
|
+
if (diff < bestDiff) {
|
|
11213
|
+
bestDiff = diff;
|
|
11214
|
+
bestZoom = zoom;
|
|
11215
|
+
}
|
|
11216
|
+
}
|
|
11217
|
+
return bestZoom;
|
|
11218
|
+
}
|
|
11219
|
+
/**
|
|
11220
|
+
* Get plugin statistics
|
|
11221
|
+
*/
|
|
11222
|
+
getStats() {
|
|
11223
|
+
return {
|
|
11224
|
+
resources: Object.keys(this.resources).length,
|
|
11225
|
+
configurations: Object.entries(this.resources).map(([name, config]) => ({
|
|
11226
|
+
resource: name,
|
|
11227
|
+
latField: config.latField,
|
|
11228
|
+
lonField: config.lonField,
|
|
11229
|
+
precision: config.precision,
|
|
11230
|
+
cellSize: `~${this._getPrecisionDistance(config.precision)}km`
|
|
11231
|
+
}))
|
|
11232
|
+
};
|
|
11233
|
+
}
|
|
11234
|
+
/**
|
|
11235
|
+
* Uninstall the plugin
|
|
11236
|
+
*/
|
|
11237
|
+
async uninstall() {
|
|
11238
|
+
if (this.verbose) {
|
|
11239
|
+
console.log("[GeoPlugin] Uninstalled");
|
|
11240
|
+
}
|
|
11241
|
+
this.emit("uninstalled", {
|
|
11242
|
+
plugin: "GeoPlugin"
|
|
11243
|
+
});
|
|
11244
|
+
await super.uninstall();
|
|
11245
|
+
}
|
|
11246
|
+
}
|
|
11247
|
+
|
|
10648
11248
|
class MetricsPlugin extends Plugin {
|
|
10649
11249
|
constructor(options = {}) {
|
|
10650
11250
|
super();
|
|
@@ -20454,7 +21054,7 @@ class Database extends EventEmitter {
|
|
|
20454
21054
|
this.id = idGenerator(7);
|
|
20455
21055
|
this.version = "1";
|
|
20456
21056
|
this.s3dbVersion = (() => {
|
|
20457
|
-
const [ok, err, version] = tryFn(() => true ? "12.2.
|
|
21057
|
+
const [ok, err, version] = tryFn(() => true ? "12.2.3" : "latest");
|
|
20458
21058
|
return ok ? version : "latest";
|
|
20459
21059
|
})();
|
|
20460
21060
|
this._resourcesMap = {};
|
|
@@ -35369,6 +35969,532 @@ class TfStatePlugin extends Plugin {
|
|
|
35369
35969
|
}
|
|
35370
35970
|
}
|
|
35371
35971
|
|
|
35972
|
+
const GRANULARITIES = {
|
|
35973
|
+
minute: {
|
|
35974
|
+
threshold: 3600,
|
|
35975
|
+
// TTL < 1 hour
|
|
35976
|
+
interval: 1e4,
|
|
35977
|
+
// Check every 10 seconds
|
|
35978
|
+
cohortsToCheck: 3,
|
|
35979
|
+
// Check last 3 minutes
|
|
35980
|
+
cohortFormat: (date) => date.toISOString().substring(0, 16)
|
|
35981
|
+
// '2024-10-25T14:30'
|
|
35982
|
+
},
|
|
35983
|
+
hour: {
|
|
35984
|
+
threshold: 86400,
|
|
35985
|
+
// TTL < 24 hours
|
|
35986
|
+
interval: 6e5,
|
|
35987
|
+
// Check every 10 minutes
|
|
35988
|
+
cohortsToCheck: 2,
|
|
35989
|
+
// Check last 2 hours
|
|
35990
|
+
cohortFormat: (date) => date.toISOString().substring(0, 13)
|
|
35991
|
+
// '2024-10-25T14'
|
|
35992
|
+
},
|
|
35993
|
+
day: {
|
|
35994
|
+
threshold: 2592e3,
|
|
35995
|
+
// TTL < 30 days
|
|
35996
|
+
interval: 36e5,
|
|
35997
|
+
// Check every 1 hour
|
|
35998
|
+
cohortsToCheck: 2,
|
|
35999
|
+
// Check last 2 days
|
|
36000
|
+
cohortFormat: (date) => date.toISOString().substring(0, 10)
|
|
36001
|
+
// '2024-10-25'
|
|
36002
|
+
},
|
|
36003
|
+
week: {
|
|
36004
|
+
threshold: Infinity,
|
|
36005
|
+
// TTL >= 30 days
|
|
36006
|
+
interval: 864e5,
|
|
36007
|
+
// Check every 24 hours
|
|
36008
|
+
cohortsToCheck: 2,
|
|
36009
|
+
// Check last 2 weeks
|
|
36010
|
+
cohortFormat: (date) => {
|
|
36011
|
+
const year = date.getUTCFullYear();
|
|
36012
|
+
const week = getWeekNumber(date);
|
|
36013
|
+
return `${year}-W${String(week).padStart(2, "0")}`;
|
|
36014
|
+
}
|
|
36015
|
+
}
|
|
36016
|
+
};
|
|
36017
|
+
function getWeekNumber(date) {
|
|
36018
|
+
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
|
|
36019
|
+
const dayNum = d.getUTCDay() || 7;
|
|
36020
|
+
d.setUTCDate(d.getUTCDate() + 4 - dayNum);
|
|
36021
|
+
const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
|
|
36022
|
+
return Math.ceil(((d - yearStart) / 864e5 + 1) / 7);
|
|
36023
|
+
}
|
|
36024
|
+
function detectGranularity(ttl) {
|
|
36025
|
+
if (!ttl) return "day";
|
|
36026
|
+
if (ttl < GRANULARITIES.minute.threshold) return "minute";
|
|
36027
|
+
if (ttl < GRANULARITIES.hour.threshold) return "hour";
|
|
36028
|
+
if (ttl < GRANULARITIES.day.threshold) return "day";
|
|
36029
|
+
return "week";
|
|
36030
|
+
}
|
|
36031
|
+
function getExpiredCohorts(granularity, count) {
|
|
36032
|
+
const config = GRANULARITIES[granularity];
|
|
36033
|
+
const cohorts = [];
|
|
36034
|
+
const now = /* @__PURE__ */ new Date();
|
|
36035
|
+
for (let i = 0; i < count; i++) {
|
|
36036
|
+
let checkDate;
|
|
36037
|
+
switch (granularity) {
|
|
36038
|
+
case "minute":
|
|
36039
|
+
checkDate = new Date(now.getTime() - i * 6e4);
|
|
36040
|
+
break;
|
|
36041
|
+
case "hour":
|
|
36042
|
+
checkDate = new Date(now.getTime() - i * 36e5);
|
|
36043
|
+
break;
|
|
36044
|
+
case "day":
|
|
36045
|
+
checkDate = new Date(now.getTime() - i * 864e5);
|
|
36046
|
+
break;
|
|
36047
|
+
case "week":
|
|
36048
|
+
checkDate = new Date(now.getTime() - i * 6048e5);
|
|
36049
|
+
break;
|
|
36050
|
+
}
|
|
36051
|
+
cohorts.push(config.cohortFormat(checkDate));
|
|
36052
|
+
}
|
|
36053
|
+
return cohorts;
|
|
36054
|
+
}
|
|
36055
|
+
class TTLPlugin extends Plugin {
|
|
36056
|
+
constructor(config = {}) {
|
|
36057
|
+
super(config);
|
|
36058
|
+
this.verbose = config.verbose !== void 0 ? config.verbose : false;
|
|
36059
|
+
this.resources = config.resources || {};
|
|
36060
|
+
this.batchSize = config.batchSize || 100;
|
|
36061
|
+
this.stats = {
|
|
36062
|
+
totalScans: 0,
|
|
36063
|
+
totalExpired: 0,
|
|
36064
|
+
totalDeleted: 0,
|
|
36065
|
+
totalArchived: 0,
|
|
36066
|
+
totalSoftDeleted: 0,
|
|
36067
|
+
totalCallbacks: 0,
|
|
36068
|
+
totalErrors: 0,
|
|
36069
|
+
lastScanAt: null,
|
|
36070
|
+
lastScanDuration: 0
|
|
36071
|
+
};
|
|
36072
|
+
this.intervals = [];
|
|
36073
|
+
this.isRunning = false;
|
|
36074
|
+
this.expirationIndex = null;
|
|
36075
|
+
}
|
|
36076
|
+
/**
|
|
36077
|
+
* Install the plugin
|
|
36078
|
+
*/
|
|
36079
|
+
async install(database) {
|
|
36080
|
+
await super.install(database);
|
|
36081
|
+
for (const [resourceName, config] of Object.entries(this.resources)) {
|
|
36082
|
+
this._validateResourceConfig(resourceName, config);
|
|
36083
|
+
}
|
|
36084
|
+
await this._createExpirationIndex();
|
|
36085
|
+
for (const [resourceName, config] of Object.entries(this.resources)) {
|
|
36086
|
+
this._setupResourceHooks(resourceName, config);
|
|
36087
|
+
}
|
|
36088
|
+
this._startIntervals();
|
|
36089
|
+
if (this.verbose) {
|
|
36090
|
+
console.log(`[TTLPlugin] Installed with ${Object.keys(this.resources).length} resources`);
|
|
36091
|
+
}
|
|
36092
|
+
this.emit("installed", {
|
|
36093
|
+
plugin: "TTLPlugin",
|
|
36094
|
+
resources: Object.keys(this.resources)
|
|
36095
|
+
});
|
|
36096
|
+
}
|
|
36097
|
+
/**
|
|
36098
|
+
* Validate resource configuration
|
|
36099
|
+
*/
|
|
36100
|
+
_validateResourceConfig(resourceName, config) {
|
|
36101
|
+
if (!config.ttl && !config.field) {
|
|
36102
|
+
throw new Error(
|
|
36103
|
+
`[TTLPlugin] Resource "${resourceName}" must have either "ttl" (seconds) or "field" (timestamp field name)`
|
|
36104
|
+
);
|
|
36105
|
+
}
|
|
36106
|
+
const validStrategies = ["soft-delete", "hard-delete", "archive", "callback"];
|
|
36107
|
+
if (!config.onExpire || !validStrategies.includes(config.onExpire)) {
|
|
36108
|
+
throw new Error(
|
|
36109
|
+
`[TTLPlugin] Resource "${resourceName}" must have an "onExpire" value. Valid options: ${validStrategies.join(", ")}`
|
|
36110
|
+
);
|
|
36111
|
+
}
|
|
36112
|
+
if (config.onExpire === "soft-delete" && !config.deleteField) {
|
|
36113
|
+
config.deleteField = "deletedat";
|
|
36114
|
+
}
|
|
36115
|
+
if (config.onExpire === "archive" && !config.archiveResource) {
|
|
36116
|
+
throw new Error(
|
|
36117
|
+
`[TTLPlugin] Resource "${resourceName}" with onExpire="archive" must have an "archiveResource" specified`
|
|
36118
|
+
);
|
|
36119
|
+
}
|
|
36120
|
+
if (config.onExpire === "callback" && typeof config.callback !== "function") {
|
|
36121
|
+
throw new Error(
|
|
36122
|
+
`[TTLPlugin] Resource "${resourceName}" with onExpire="callback" must have a "callback" function`
|
|
36123
|
+
);
|
|
36124
|
+
}
|
|
36125
|
+
if (!config.field) {
|
|
36126
|
+
config.field = "_createdAt";
|
|
36127
|
+
}
|
|
36128
|
+
if (config.field === "_createdAt" && this.database) {
|
|
36129
|
+
const resource = this.database.resources[resourceName];
|
|
36130
|
+
if (resource && resource.config && resource.config.timestamps === false) {
|
|
36131
|
+
console.warn(
|
|
36132
|
+
`[TTLPlugin] WARNING: Resource "${resourceName}" uses TTL with field "_createdAt" but timestamps are disabled. TTL will be calculated from indexing time, not creation time.`
|
|
36133
|
+
);
|
|
36134
|
+
}
|
|
36135
|
+
}
|
|
36136
|
+
config.granularity = detectGranularity(config.ttl);
|
|
36137
|
+
}
|
|
36138
|
+
/**
|
|
36139
|
+
* Create expiration index (plugin resource)
|
|
36140
|
+
*/
|
|
36141
|
+
async _createExpirationIndex() {
|
|
36142
|
+
this.expirationIndex = await this.database.createResource({
|
|
36143
|
+
name: "plg_ttl_expiration_index",
|
|
36144
|
+
attributes: {
|
|
36145
|
+
resourceName: "string|required",
|
|
36146
|
+
recordId: "string|required",
|
|
36147
|
+
expiresAtCohort: "string|required",
|
|
36148
|
+
expiresAtTimestamp: "number|required",
|
|
36149
|
+
// Exact expiration timestamp for precise checking
|
|
36150
|
+
granularity: "string|required",
|
|
36151
|
+
createdAt: "number"
|
|
36152
|
+
},
|
|
36153
|
+
partitions: {
|
|
36154
|
+
byExpiresAtCohort: {
|
|
36155
|
+
fields: { expiresAtCohort: "string" }
|
|
36156
|
+
}
|
|
36157
|
+
},
|
|
36158
|
+
asyncPartitions: false
|
|
36159
|
+
// Sync partitions for deterministic behavior
|
|
36160
|
+
});
|
|
36161
|
+
if (this.verbose) {
|
|
36162
|
+
console.log("[TTLPlugin] Created expiration index with partition");
|
|
36163
|
+
}
|
|
36164
|
+
}
|
|
36165
|
+
/**
|
|
36166
|
+
* Setup hooks for a resource
|
|
36167
|
+
*/
|
|
36168
|
+
_setupResourceHooks(resourceName, config) {
|
|
36169
|
+
if (!this.database.resources[resourceName]) {
|
|
36170
|
+
if (this.verbose) {
|
|
36171
|
+
console.warn(`[TTLPlugin] Resource "${resourceName}" not found, skipping hooks`);
|
|
36172
|
+
}
|
|
36173
|
+
return;
|
|
36174
|
+
}
|
|
36175
|
+
const resource = this.database.resources[resourceName];
|
|
36176
|
+
if (typeof resource.insert !== "function" || typeof resource.delete !== "function") {
|
|
36177
|
+
if (this.verbose) {
|
|
36178
|
+
console.warn(`[TTLPlugin] Resource "${resourceName}" missing insert/delete methods, skipping hooks`);
|
|
36179
|
+
}
|
|
36180
|
+
return;
|
|
36181
|
+
}
|
|
36182
|
+
this.addMiddleware(resource, "insert", async (next, data, options) => {
|
|
36183
|
+
const result = await next(data, options);
|
|
36184
|
+
await this._addToIndex(resourceName, result, config);
|
|
36185
|
+
return result;
|
|
36186
|
+
});
|
|
36187
|
+
this.addMiddleware(resource, "delete", async (next, id, options) => {
|
|
36188
|
+
const result = await next(id, options);
|
|
36189
|
+
await this._removeFromIndex(resourceName, id);
|
|
36190
|
+
return result;
|
|
36191
|
+
});
|
|
36192
|
+
if (this.verbose) {
|
|
36193
|
+
console.log(`[TTLPlugin] Setup hooks for resource "${resourceName}"`);
|
|
36194
|
+
}
|
|
36195
|
+
}
|
|
36196
|
+
/**
|
|
36197
|
+
* Add record to expiration index
|
|
36198
|
+
*/
|
|
36199
|
+
async _addToIndex(resourceName, record, config) {
|
|
36200
|
+
try {
|
|
36201
|
+
let baseTime = record[config.field];
|
|
36202
|
+
if (!baseTime && config.field === "_createdAt") {
|
|
36203
|
+
baseTime = Date.now();
|
|
36204
|
+
}
|
|
36205
|
+
if (!baseTime) {
|
|
36206
|
+
if (this.verbose) {
|
|
36207
|
+
console.warn(
|
|
36208
|
+
`[TTLPlugin] Record ${record.id} in ${resourceName} missing field "${config.field}", skipping index`
|
|
36209
|
+
);
|
|
36210
|
+
}
|
|
36211
|
+
return;
|
|
36212
|
+
}
|
|
36213
|
+
const baseTimestamp = typeof baseTime === "number" ? baseTime : new Date(baseTime).getTime();
|
|
36214
|
+
const expiresAt = config.ttl ? new Date(baseTimestamp + config.ttl * 1e3) : new Date(baseTimestamp);
|
|
36215
|
+
const cohortConfig = GRANULARITIES[config.granularity];
|
|
36216
|
+
const cohort = cohortConfig.cohortFormat(expiresAt);
|
|
36217
|
+
const indexId = `${resourceName}:${record.id}`;
|
|
36218
|
+
await this.expirationIndex.insert({
|
|
36219
|
+
id: indexId,
|
|
36220
|
+
resourceName,
|
|
36221
|
+
recordId: record.id,
|
|
36222
|
+
expiresAtCohort: cohort,
|
|
36223
|
+
expiresAtTimestamp: expiresAt.getTime(),
|
|
36224
|
+
// Store exact timestamp for precise checking
|
|
36225
|
+
granularity: config.granularity,
|
|
36226
|
+
createdAt: Date.now()
|
|
36227
|
+
});
|
|
36228
|
+
if (this.verbose) {
|
|
36229
|
+
console.log(
|
|
36230
|
+
`[TTLPlugin] Added ${resourceName}:${record.id} to index (cohort: ${cohort}, granularity: ${config.granularity})`
|
|
36231
|
+
);
|
|
36232
|
+
}
|
|
36233
|
+
} catch (error) {
|
|
36234
|
+
console.error(`[TTLPlugin] Error adding to index:`, error);
|
|
36235
|
+
this.stats.totalErrors++;
|
|
36236
|
+
}
|
|
36237
|
+
}
|
|
36238
|
+
/**
|
|
36239
|
+
* Remove record from expiration index (O(1) using deterministic ID)
|
|
36240
|
+
*/
|
|
36241
|
+
async _removeFromIndex(resourceName, recordId) {
|
|
36242
|
+
try {
|
|
36243
|
+
const indexId = `${resourceName}:${recordId}`;
|
|
36244
|
+
const [ok, err] = await tryFn(() => this.expirationIndex.delete(indexId));
|
|
36245
|
+
if (this.verbose && ok) {
|
|
36246
|
+
console.log(`[TTLPlugin] Removed index entry for ${resourceName}:${recordId}`);
|
|
36247
|
+
}
|
|
36248
|
+
if (!ok && err?.code !== "NoSuchKey") {
|
|
36249
|
+
throw err;
|
|
36250
|
+
}
|
|
36251
|
+
} catch (error) {
|
|
36252
|
+
console.error(`[TTLPlugin] Error removing from index:`, error);
|
|
36253
|
+
}
|
|
36254
|
+
}
|
|
36255
|
+
/**
|
|
36256
|
+
* Start interval-based cleanup for each granularity
|
|
36257
|
+
*/
|
|
36258
|
+
_startIntervals() {
|
|
36259
|
+
const byGranularity = {
|
|
36260
|
+
minute: [],
|
|
36261
|
+
hour: [],
|
|
36262
|
+
day: [],
|
|
36263
|
+
week: []
|
|
36264
|
+
};
|
|
36265
|
+
for (const [name, config] of Object.entries(this.resources)) {
|
|
36266
|
+
byGranularity[config.granularity].push({ name, config });
|
|
36267
|
+
}
|
|
36268
|
+
for (const [granularity, resources] of Object.entries(byGranularity)) {
|
|
36269
|
+
if (resources.length === 0) continue;
|
|
36270
|
+
const granularityConfig = GRANULARITIES[granularity];
|
|
36271
|
+
const handle = setInterval(
|
|
36272
|
+
() => this._cleanupGranularity(granularity, resources),
|
|
36273
|
+
granularityConfig.interval
|
|
36274
|
+
);
|
|
36275
|
+
this.intervals.push(handle);
|
|
36276
|
+
if (this.verbose) {
|
|
36277
|
+
console.log(
|
|
36278
|
+
`[TTLPlugin] Started ${granularity} interval (${granularityConfig.interval}ms) for ${resources.length} resources`
|
|
36279
|
+
);
|
|
36280
|
+
}
|
|
36281
|
+
}
|
|
36282
|
+
this.isRunning = true;
|
|
36283
|
+
}
|
|
36284
|
+
/**
|
|
36285
|
+
* Stop all intervals
|
|
36286
|
+
*/
|
|
36287
|
+
_stopIntervals() {
|
|
36288
|
+
for (const handle of this.intervals) {
|
|
36289
|
+
clearInterval(handle);
|
|
36290
|
+
}
|
|
36291
|
+
this.intervals = [];
|
|
36292
|
+
this.isRunning = false;
|
|
36293
|
+
if (this.verbose) {
|
|
36294
|
+
console.log("[TTLPlugin] Stopped all intervals");
|
|
36295
|
+
}
|
|
36296
|
+
}
|
|
36297
|
+
/**
|
|
36298
|
+
* Cleanup expired records for a specific granularity
|
|
36299
|
+
*/
|
|
36300
|
+
async _cleanupGranularity(granularity, resources) {
|
|
36301
|
+
const startTime = Date.now();
|
|
36302
|
+
this.stats.totalScans++;
|
|
36303
|
+
try {
|
|
36304
|
+
const granularityConfig = GRANULARITIES[granularity];
|
|
36305
|
+
const cohorts = getExpiredCohorts(granularity, granularityConfig.cohortsToCheck);
|
|
36306
|
+
if (this.verbose) {
|
|
36307
|
+
console.log(`[TTLPlugin] Cleaning ${granularity} granularity, checking cohorts:`, cohorts);
|
|
36308
|
+
}
|
|
36309
|
+
for (const cohort of cohorts) {
|
|
36310
|
+
const expired = await this.expirationIndex.listPartition({
|
|
36311
|
+
partition: "byExpiresAtCohort",
|
|
36312
|
+
partitionValues: { expiresAtCohort: cohort }
|
|
36313
|
+
});
|
|
36314
|
+
const resourceNames = new Set(resources.map((r) => r.name));
|
|
36315
|
+
const filtered = expired.filter((e) => resourceNames.has(e.resourceName));
|
|
36316
|
+
if (this.verbose && filtered.length > 0) {
|
|
36317
|
+
console.log(`[TTLPlugin] Found ${filtered.length} expired records in cohort ${cohort}`);
|
|
36318
|
+
}
|
|
36319
|
+
for (let i = 0; i < filtered.length; i += this.batchSize) {
|
|
36320
|
+
const batch = filtered.slice(i, i + this.batchSize);
|
|
36321
|
+
for (const entry of batch) {
|
|
36322
|
+
const config = this.resources[entry.resourceName];
|
|
36323
|
+
await this._processExpiredEntry(entry, config);
|
|
36324
|
+
}
|
|
36325
|
+
}
|
|
36326
|
+
}
|
|
36327
|
+
this.stats.lastScanAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
36328
|
+
this.stats.lastScanDuration = Date.now() - startTime;
|
|
36329
|
+
this.emit("scanCompleted", {
|
|
36330
|
+
granularity,
|
|
36331
|
+
duration: this.stats.lastScanDuration,
|
|
36332
|
+
cohorts
|
|
36333
|
+
});
|
|
36334
|
+
} catch (error) {
|
|
36335
|
+
console.error(`[TTLPlugin] Error in ${granularity} cleanup:`, error);
|
|
36336
|
+
this.stats.totalErrors++;
|
|
36337
|
+
this.emit("cleanupError", { granularity, error });
|
|
36338
|
+
}
|
|
36339
|
+
}
|
|
36340
|
+
/**
|
|
36341
|
+
* Process a single expired index entry
|
|
36342
|
+
*/
|
|
36343
|
+
async _processExpiredEntry(entry, config) {
|
|
36344
|
+
try {
|
|
36345
|
+
if (!this.database.resources[entry.resourceName]) {
|
|
36346
|
+
if (this.verbose) {
|
|
36347
|
+
console.warn(`[TTLPlugin] Resource "${entry.resourceName}" not found during cleanup, skipping`);
|
|
36348
|
+
}
|
|
36349
|
+
return;
|
|
36350
|
+
}
|
|
36351
|
+
const resource = this.database.resources[entry.resourceName];
|
|
36352
|
+
const [ok, err, record] = await tryFn(() => resource.get(entry.recordId));
|
|
36353
|
+
if (!ok || !record) {
|
|
36354
|
+
await this.expirationIndex.delete(entry.id);
|
|
36355
|
+
return;
|
|
36356
|
+
}
|
|
36357
|
+
if (entry.expiresAtTimestamp && Date.now() < entry.expiresAtTimestamp) {
|
|
36358
|
+
return;
|
|
36359
|
+
}
|
|
36360
|
+
switch (config.onExpire) {
|
|
36361
|
+
case "soft-delete":
|
|
36362
|
+
await this._softDelete(resource, record, config);
|
|
36363
|
+
this.stats.totalSoftDeleted++;
|
|
36364
|
+
break;
|
|
36365
|
+
case "hard-delete":
|
|
36366
|
+
await this._hardDelete(resource, record);
|
|
36367
|
+
this.stats.totalDeleted++;
|
|
36368
|
+
break;
|
|
36369
|
+
case "archive":
|
|
36370
|
+
await this._archive(resource, record, config);
|
|
36371
|
+
this.stats.totalArchived++;
|
|
36372
|
+
this.stats.totalDeleted++;
|
|
36373
|
+
break;
|
|
36374
|
+
case "callback":
|
|
36375
|
+
const shouldDelete = await config.callback(record, resource);
|
|
36376
|
+
this.stats.totalCallbacks++;
|
|
36377
|
+
if (shouldDelete) {
|
|
36378
|
+
await this._hardDelete(resource, record);
|
|
36379
|
+
this.stats.totalDeleted++;
|
|
36380
|
+
}
|
|
36381
|
+
break;
|
|
36382
|
+
}
|
|
36383
|
+
await this.expirationIndex.delete(entry.id);
|
|
36384
|
+
this.stats.totalExpired++;
|
|
36385
|
+
this.emit("recordExpired", { resource: entry.resourceName, record });
|
|
36386
|
+
} catch (error) {
|
|
36387
|
+
console.error(`[TTLPlugin] Error processing expired entry:`, error);
|
|
36388
|
+
this.stats.totalErrors++;
|
|
36389
|
+
}
|
|
36390
|
+
}
|
|
36391
|
+
/**
|
|
36392
|
+
* Soft delete: Mark record as deleted
|
|
36393
|
+
*/
|
|
36394
|
+
async _softDelete(resource, record, config) {
|
|
36395
|
+
const deleteField = config.deleteField || "deletedat";
|
|
36396
|
+
const updates = {
|
|
36397
|
+
[deleteField]: (/* @__PURE__ */ new Date()).toISOString(),
|
|
36398
|
+
isdeleted: "true"
|
|
36399
|
+
// Add isdeleted field for partition compatibility
|
|
36400
|
+
};
|
|
36401
|
+
await resource.update(record.id, updates);
|
|
36402
|
+
if (this.verbose) {
|
|
36403
|
+
console.log(`[TTLPlugin] Soft-deleted record ${record.id} in ${resource.name}`);
|
|
36404
|
+
}
|
|
36405
|
+
}
|
|
36406
|
+
/**
|
|
36407
|
+
* Hard delete: Remove record from S3
|
|
36408
|
+
*/
|
|
36409
|
+
async _hardDelete(resource, record) {
|
|
36410
|
+
await resource.delete(record.id);
|
|
36411
|
+
if (this.verbose) {
|
|
36412
|
+
console.log(`[TTLPlugin] Hard-deleted record ${record.id} in ${resource.name}`);
|
|
36413
|
+
}
|
|
36414
|
+
}
|
|
36415
|
+
/**
|
|
36416
|
+
* Archive: Copy to another resource then delete
|
|
36417
|
+
*/
|
|
36418
|
+
async _archive(resource, record, config) {
|
|
36419
|
+
if (!this.database.resources[config.archiveResource]) {
|
|
36420
|
+
throw new Error(`Archive resource "${config.archiveResource}" not found`);
|
|
36421
|
+
}
|
|
36422
|
+
const archiveResource = this.database.resources[config.archiveResource];
|
|
36423
|
+
const archiveData = {};
|
|
36424
|
+
for (const [key, value] of Object.entries(record)) {
|
|
36425
|
+
if (!key.startsWith("_")) {
|
|
36426
|
+
archiveData[key] = value;
|
|
36427
|
+
}
|
|
36428
|
+
}
|
|
36429
|
+
archiveData.archivedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
36430
|
+
archiveData.archivedFrom = resource.name;
|
|
36431
|
+
archiveData.originalId = record.id;
|
|
36432
|
+
if (!config.keepOriginalId) {
|
|
36433
|
+
delete archiveData.id;
|
|
36434
|
+
}
|
|
36435
|
+
await archiveResource.insert(archiveData);
|
|
36436
|
+
await resource.delete(record.id);
|
|
36437
|
+
if (this.verbose) {
|
|
36438
|
+
console.log(`[TTLPlugin] Archived record ${record.id} from ${resource.name} to ${config.archiveResource}`);
|
|
36439
|
+
}
|
|
36440
|
+
}
|
|
36441
|
+
/**
|
|
36442
|
+
* Manual cleanup of a specific resource
|
|
36443
|
+
*/
|
|
36444
|
+
async cleanupResource(resourceName) {
|
|
36445
|
+
const config = this.resources[resourceName];
|
|
36446
|
+
if (!config) {
|
|
36447
|
+
throw new Error(`Resource "${resourceName}" not configured in TTLPlugin`);
|
|
36448
|
+
}
|
|
36449
|
+
const granularity = config.granularity;
|
|
36450
|
+
await this._cleanupGranularity(granularity, [{ name: resourceName, config }]);
|
|
36451
|
+
return {
|
|
36452
|
+
resource: resourceName,
|
|
36453
|
+
granularity
|
|
36454
|
+
};
|
|
36455
|
+
}
|
|
36456
|
+
/**
|
|
36457
|
+
* Manual cleanup of all resources
|
|
36458
|
+
*/
|
|
36459
|
+
async runCleanup() {
|
|
36460
|
+
const byGranularity = {
|
|
36461
|
+
minute: [],
|
|
36462
|
+
hour: [],
|
|
36463
|
+
day: [],
|
|
36464
|
+
week: []
|
|
36465
|
+
};
|
|
36466
|
+
for (const [name, config] of Object.entries(this.resources)) {
|
|
36467
|
+
byGranularity[config.granularity].push({ name, config });
|
|
36468
|
+
}
|
|
36469
|
+
for (const [granularity, resources] of Object.entries(byGranularity)) {
|
|
36470
|
+
if (resources.length > 0) {
|
|
36471
|
+
await this._cleanupGranularity(granularity, resources);
|
|
36472
|
+
}
|
|
36473
|
+
}
|
|
36474
|
+
}
|
|
36475
|
+
/**
|
|
36476
|
+
* Get plugin statistics
|
|
36477
|
+
*/
|
|
36478
|
+
getStats() {
|
|
36479
|
+
return {
|
|
36480
|
+
...this.stats,
|
|
36481
|
+
resources: Object.keys(this.resources).length,
|
|
36482
|
+
isRunning: this.isRunning,
|
|
36483
|
+
intervals: this.intervals.length
|
|
36484
|
+
};
|
|
36485
|
+
}
|
|
36486
|
+
/**
|
|
36487
|
+
* Uninstall the plugin
|
|
36488
|
+
*/
|
|
36489
|
+
async uninstall() {
|
|
36490
|
+
this._stopIntervals();
|
|
36491
|
+
await super.uninstall();
|
|
36492
|
+
if (this.verbose) {
|
|
36493
|
+
console.log("[TTLPlugin] Uninstalled");
|
|
36494
|
+
}
|
|
36495
|
+
}
|
|
36496
|
+
}
|
|
36497
|
+
|
|
35372
36498
|
function cosineDistance(a, b) {
|
|
35373
36499
|
if (a.length !== b.length) {
|
|
35374
36500
|
throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
|
|
@@ -37170,6 +38296,7 @@ exports.Factory = Factory;
|
|
|
37170
38296
|
exports.FilesystemBackupDriver = FilesystemBackupDriver;
|
|
37171
38297
|
exports.FilesystemCache = FilesystemCache;
|
|
37172
38298
|
exports.FullTextPlugin = FullTextPlugin;
|
|
38299
|
+
exports.GeoPlugin = GeoPlugin;
|
|
37173
38300
|
exports.InvalidResourceItem = InvalidResourceItem;
|
|
37174
38301
|
exports.MemoryCache = MemoryCache;
|
|
37175
38302
|
exports.MetadataLimitError = MetadataLimitError;
|
|
@@ -37217,6 +38344,7 @@ exports.SqsConsumer = SqsConsumer;
|
|
|
37217
38344
|
exports.SqsReplicator = SqsReplicator;
|
|
37218
38345
|
exports.StateMachinePlugin = StateMachinePlugin;
|
|
37219
38346
|
exports.StreamError = StreamError;
|
|
38347
|
+
exports.TTLPlugin = TTLPlugin;
|
|
37220
38348
|
exports.TfStatePlugin = TfStatePlugin;
|
|
37221
38349
|
exports.TursoReplicator = TursoReplicator;
|
|
37222
38350
|
exports.UnknownError = UnknownError;
|