s3db.js 11.3.2 → 12.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -8
- package/dist/s3db.cjs.js +36945 -15510
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.d.ts +66 -1
- package/dist/s3db.es.js +36914 -15534
- package/dist/s3db.es.js.map +1 -1
- package/mcp/entrypoint.js +58 -0
- package/mcp/tools/documentation.js +434 -0
- package/mcp/tools/index.js +4 -0
- package/package.json +35 -15
- package/src/behaviors/user-managed.js +13 -6
- package/src/client.class.js +79 -49
- package/src/concerns/base62.js +85 -0
- package/src/concerns/dictionary-encoding.js +294 -0
- package/src/concerns/geo-encoding.js +256 -0
- package/src/concerns/high-performance-inserter.js +34 -30
- package/src/concerns/ip.js +325 -0
- package/src/concerns/metadata-encoding.js +345 -66
- package/src/concerns/money.js +193 -0
- package/src/concerns/partition-queue.js +7 -4
- package/src/concerns/plugin-storage.js +97 -47
- package/src/database.class.js +76 -74
- package/src/errors.js +0 -4
- package/src/plugins/api/auth/api-key-auth.js +88 -0
- package/src/plugins/api/auth/basic-auth.js +154 -0
- package/src/plugins/api/auth/index.js +112 -0
- package/src/plugins/api/auth/jwt-auth.js +169 -0
- package/src/plugins/api/index.js +544 -0
- package/src/plugins/api/middlewares/index.js +15 -0
- package/src/plugins/api/middlewares/validator.js +185 -0
- package/src/plugins/api/routes/auth-routes.js +241 -0
- package/src/plugins/api/routes/resource-routes.js +304 -0
- package/src/plugins/api/server.js +354 -0
- package/src/plugins/api/utils/error-handler.js +147 -0
- package/src/plugins/api/utils/openapi-generator.js +1240 -0
- package/src/plugins/api/utils/response-formatter.js +218 -0
- package/src/plugins/backup/streaming-exporter.js +132 -0
- package/src/plugins/backup.plugin.js +103 -50
- package/src/plugins/cache/s3-cache.class.js +95 -47
- package/src/plugins/cache.plugin.js +107 -9
- package/src/plugins/concerns/plugin-dependencies.js +313 -0
- package/src/plugins/concerns/prometheus-formatter.js +255 -0
- package/src/plugins/consumers/rabbitmq-consumer.js +4 -0
- package/src/plugins/consumers/sqs-consumer.js +4 -0
- package/src/plugins/costs.plugin.js +255 -39
- package/src/plugins/eventual-consistency/helpers.js +15 -1
- package/src/plugins/geo.plugin.js +873 -0
- package/src/plugins/importer/index.js +1020 -0
- package/src/plugins/index.js +11 -0
- package/src/plugins/metrics.plugin.js +163 -4
- package/src/plugins/queue-consumer.plugin.js +6 -27
- package/src/plugins/relation.errors.js +139 -0
- package/src/plugins/relation.plugin.js +1242 -0
- package/src/plugins/replicator.plugin.js +2 -1
- package/src/plugins/replicators/bigquery-replicator.class.js +180 -8
- package/src/plugins/replicators/dynamodb-replicator.class.js +383 -0
- package/src/plugins/replicators/index.js +28 -3
- package/src/plugins/replicators/mongodb-replicator.class.js +391 -0
- package/src/plugins/replicators/mysql-replicator.class.js +558 -0
- package/src/plugins/replicators/planetscale-replicator.class.js +409 -0
- package/src/plugins/replicators/postgres-replicator.class.js +182 -7
- package/src/plugins/replicators/s3db-replicator.class.js +1 -12
- package/src/plugins/replicators/schema-sync.helper.js +601 -0
- package/src/plugins/replicators/sqs-replicator.class.js +11 -9
- package/src/plugins/replicators/turso-replicator.class.js +416 -0
- package/src/plugins/replicators/webhook-replicator.class.js +612 -0
- package/src/plugins/state-machine.plugin.js +122 -68
- package/src/plugins/tfstate/README.md +745 -0
- package/src/plugins/tfstate/base-driver.js +80 -0
- package/src/plugins/tfstate/errors.js +112 -0
- package/src/plugins/tfstate/filesystem-driver.js +129 -0
- package/src/plugins/tfstate/index.js +2660 -0
- package/src/plugins/tfstate/s3-driver.js +192 -0
- package/src/plugins/ttl.plugin.js +536 -0
- package/src/resource.class.js +315 -36
- package/src/s3db.d.ts +66 -1
- package/src/schema.class.js +366 -32
- package/SECURITY.md +0 -76
- package/src/partition-drivers/base-partition-driver.js +0 -106
- package/src/partition-drivers/index.js +0 -66
- package/src/partition-drivers/memory-partition-driver.js +0 -289
- package/src/partition-drivers/sqs-partition-driver.js +0 -337
- package/src/partition-drivers/sync-partition-driver.js +0 -38
|
@@ -0,0 +1,873 @@
|
|
|
1
|
+
import Plugin from "./plugin.class.js";
|
|
2
|
+
import tryFn from "../concerns/try-fn.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* GeoPlugin - Geospatial Queries and Location-Based Features
|
|
6
|
+
*
|
|
7
|
+
* Provides geospatial capabilities including proximity search, bounding box queries,
|
|
8
|
+
* distance calculations, and automatic geohash partitioning for efficient location queries.
|
|
9
|
+
*
|
|
10
|
+
* === Features ===
|
|
11
|
+
* - Automatic geohash calculation and indexing
|
|
12
|
+
* - Proximity search (find nearby locations)
|
|
13
|
+
* - Bounding box queries (find within area)
|
|
14
|
+
* - Distance calculation between two points (Haversine formula)
|
|
15
|
+
* - Configurable geohash precision per resource
|
|
16
|
+
* - Automatic partition creation for efficient queries
|
|
17
|
+
* - Support for latitude/longitude fields
|
|
18
|
+
*
|
|
19
|
+
* === Configuration Example ===
|
|
20
|
+
*
|
|
21
|
+
* new GeoPlugin({
|
|
22
|
+
* resources: {
|
|
23
|
+
* stores: {
|
|
24
|
+
* latField: 'latitude', // Latitude field name
|
|
25
|
+
* lonField: 'longitude', // Longitude field name
|
|
26
|
+
* precision: 5, // Geohash precision (~5km cells)
|
|
27
|
+
* addGeohash: true, // Add 'geohash' field automatically
|
|
28
|
+
* usePartitions: true, // Create geohash partitions for efficient queries
|
|
29
|
+
* zoomLevels: [4, 5, 6, 7] // Multi-zoom partitions (4=~20km, 5=~5km, 6=~1.2km, 7=~150m)
|
|
30
|
+
* },
|
|
31
|
+
*
|
|
32
|
+
* restaurants: {
|
|
33
|
+
* latField: 'lat',
|
|
34
|
+
* lonField: 'lng',
|
|
35
|
+
* precision: 6, // Higher precision (~1.2km cells)
|
|
36
|
+
* usePartitions: true, // Enables O(1) geohash lookups
|
|
37
|
+
* zoomLevels: [5, 6, 7, 8] // Fine-grained zooms for dense urban areas
|
|
38
|
+
* }
|
|
39
|
+
* }
|
|
40
|
+
* })
|
|
41
|
+
*
|
|
42
|
+
* // With zoomLevels, queries auto-select optimal partition based on search radius:
|
|
43
|
+
* // - Large radius (>10km): uses zoom4 (~20km cells)
|
|
44
|
+
* // - Medium radius (2-10km): uses zoom5 (~5km cells)
|
|
45
|
+
* // - Small radius (0.5-2km): uses zoom6 (~1.2km cells)
|
|
46
|
+
* // - Precise radius (<0.5km): uses zoom7 (~150m cells)
|
|
47
|
+
*
|
|
48
|
+
* === Geohash Precision ===
|
|
49
|
+
*
|
|
50
|
+
* | Precision | Cell Size | Use Case |
|
|
51
|
+
* |-----------|-----------|----------|
|
|
52
|
+
* | 4 | ~20km | Country/state level |
|
|
53
|
+
* | 5 | ~5km | City districts, delivery zones |
|
|
54
|
+
* | 6 | ~1.2km | Neighborhoods, local search |
|
|
55
|
+
* | 7 | ~150m | Street-level accuracy |
|
|
56
|
+
* | 8 | ~38m | Building-level accuracy |
|
|
57
|
+
*
|
|
58
|
+
* === Helper Methods Added to Resources ===
|
|
59
|
+
*
|
|
60
|
+
* resource.findNearby({
|
|
61
|
+
* lat: -23.5505,
|
|
62
|
+
* lon: -46.6333,
|
|
63
|
+
* radius: 10, // km
|
|
64
|
+
* limit: 20
|
|
65
|
+
* })
|
|
66
|
+
*
|
|
67
|
+
* resource.findInBounds({
|
|
68
|
+
* north: -23.5,
|
|
69
|
+
* south: -23.6,
|
|
70
|
+
* east: -46.6,
|
|
71
|
+
* west: -46.7
|
|
72
|
+
* })
|
|
73
|
+
*
|
|
74
|
+
* resource.getDistance(id1, id2) // Returns distance in km
|
|
75
|
+
*/
|
|
76
|
+
class GeoPlugin extends Plugin {
|
|
77
|
+
constructor(config = {}) {
|
|
78
|
+
super(config);
|
|
79
|
+
|
|
80
|
+
this.resources = config.resources || {};
|
|
81
|
+
this.verbose = config.verbose !== undefined ? config.verbose : false;
|
|
82
|
+
|
|
83
|
+
// Geohash base32 alphabet
|
|
84
|
+
this.base32 = '0123456789bcdefghjkmnpqrstuvwxyz';
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Install the plugin
|
|
89
|
+
*/
|
|
90
|
+
async install(database) {
|
|
91
|
+
await super.install(database);
|
|
92
|
+
|
|
93
|
+
// Validate and setup each resource
|
|
94
|
+
for (const [resourceName, config] of Object.entries(this.resources)) {
|
|
95
|
+
await this._setupResource(resourceName, config);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Watch for resources created after plugin installation
|
|
99
|
+
this.database.addHook('afterCreateResource', async (context) => {
|
|
100
|
+
const { resource, config: resourceConfig } = context;
|
|
101
|
+
const geoConfig = this.resources[resource.name];
|
|
102
|
+
|
|
103
|
+
if (geoConfig) {
|
|
104
|
+
await this._setupResource(resource.name, geoConfig);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
if (this.verbose) {
|
|
109
|
+
console.log(`[GeoPlugin] Installed with ${Object.keys(this.resources).length} resources`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
this.emit('installed', {
|
|
113
|
+
plugin: 'GeoPlugin',
|
|
114
|
+
resources: Object.keys(this.resources)
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Setup a resource with geo capabilities
|
|
120
|
+
*/
|
|
121
|
+
async _setupResource(resourceName, config) {
|
|
122
|
+
// Check if resource exists first
|
|
123
|
+
if (!this.database.resources[resourceName]) {
|
|
124
|
+
if (this.verbose) {
|
|
125
|
+
console.warn(`[GeoPlugin] Resource "${resourceName}" not found, will setup when created`);
|
|
126
|
+
}
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const resource = this.database.resource(resourceName);
|
|
131
|
+
if (!resource || typeof resource.addHook !== 'function') {
|
|
132
|
+
if (this.verbose) {
|
|
133
|
+
console.warn(`[GeoPlugin] Resource "${resourceName}" not found or invalid`);
|
|
134
|
+
}
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Validate configuration
|
|
139
|
+
if (!config.latField || !config.lonField) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
`[GeoPlugin] Resource "${resourceName}" must have "latField" and "lonField" configured`
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if (!config.precision || config.precision < 1 || config.precision > 12) {
|
|
146
|
+
config.precision = 5; // Default precision
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Store config on resource
|
|
150
|
+
resource._geoConfig = config;
|
|
151
|
+
|
|
152
|
+
// Add geohash fields to resource schema if not already present
|
|
153
|
+
// Check if lat/lon fields are optional to determine if geohash fields should also be optional
|
|
154
|
+
const latField = resource.attributes[config.latField];
|
|
155
|
+
const lonField = resource.attributes[config.lonField];
|
|
156
|
+
const isLatOptional = typeof latField === 'object' && latField.optional === true;
|
|
157
|
+
const isLonOptional = typeof lonField === 'object' && lonField.optional === true;
|
|
158
|
+
const areCoordinatesOptional = isLatOptional || isLonOptional;
|
|
159
|
+
|
|
160
|
+
const geohashType = areCoordinatesOptional ? 'string|optional' : 'string';
|
|
161
|
+
|
|
162
|
+
let needsUpdate = false;
|
|
163
|
+
const newAttributes = { ...resource.attributes };
|
|
164
|
+
|
|
165
|
+
if (config.addGeohash && !newAttributes.geohash) {
|
|
166
|
+
newAttributes.geohash = geohashType;
|
|
167
|
+
needsUpdate = true;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
if (!newAttributes._geohash) {
|
|
171
|
+
newAttributes._geohash = geohashType;
|
|
172
|
+
needsUpdate = true;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Add zoom-specific fields if using zoomLevels
|
|
176
|
+
if (config.zoomLevels && Array.isArray(config.zoomLevels)) {
|
|
177
|
+
for (const zoom of config.zoomLevels) {
|
|
178
|
+
const fieldName = `_geohash_zoom${zoom}`;
|
|
179
|
+
if (!newAttributes[fieldName]) {
|
|
180
|
+
newAttributes[fieldName] = geohashType;
|
|
181
|
+
needsUpdate = true;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Update schema if we added new fields (this regenerates field maps)
|
|
187
|
+
if (needsUpdate) {
|
|
188
|
+
resource.updateAttributes(newAttributes);
|
|
189
|
+
|
|
190
|
+
// Persist schema changes to metadata
|
|
191
|
+
if (this.database.uploadMetadataFile) {
|
|
192
|
+
await this.database.uploadMetadataFile();
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Setup geohash partitions if enabled
|
|
197
|
+
if (config.usePartitions) {
|
|
198
|
+
await this._setupPartitions(resource, config);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Add hooks for automatic geohash calculation
|
|
202
|
+
this._addHooks(resource, config);
|
|
203
|
+
|
|
204
|
+
// Add helper methods to resource
|
|
205
|
+
this._addHelperMethods(resource, config);
|
|
206
|
+
|
|
207
|
+
if (this.verbose) {
|
|
208
|
+
console.log(
|
|
209
|
+
`[GeoPlugin] Setup resource "${resourceName}" with precision ${config.precision} ` +
|
|
210
|
+
`(~${this._getPrecisionDistance(config.precision)}km cells)` +
|
|
211
|
+
(config.usePartitions ? ' [Partitions enabled]' : '')
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Setup geohash partitions for efficient spatial queries
|
|
218
|
+
* Creates multiple zoom-level partitions if zoomLevels configured
|
|
219
|
+
*/
|
|
220
|
+
async _setupPartitions(resource, config) {
|
|
221
|
+
const updatedConfig = { ...resource.config };
|
|
222
|
+
updatedConfig.partitions = updatedConfig.partitions || {};
|
|
223
|
+
|
|
224
|
+
let partitionsCreated = 0;
|
|
225
|
+
|
|
226
|
+
// If zoomLevels configured, create partition for each zoom level
|
|
227
|
+
if (config.zoomLevels && Array.isArray(config.zoomLevels)) {
|
|
228
|
+
for (const zoom of config.zoomLevels) {
|
|
229
|
+
const partitionName = `byGeohashZoom${zoom}`;
|
|
230
|
+
const fieldName = `_geohash_zoom${zoom}`;
|
|
231
|
+
|
|
232
|
+
if (!updatedConfig.partitions[partitionName]) {
|
|
233
|
+
updatedConfig.partitions[partitionName] = {
|
|
234
|
+
fields: {
|
|
235
|
+
[fieldName]: 'string'
|
|
236
|
+
}
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
partitionsCreated++;
|
|
240
|
+
|
|
241
|
+
if (this.verbose) {
|
|
242
|
+
console.log(
|
|
243
|
+
`[GeoPlugin] Created ${partitionName} partition for "${resource.name}" ` +
|
|
244
|
+
`(precision ${zoom}, ~${this._getPrecisionDistance(zoom)}km cells)`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
} else {
|
|
250
|
+
// Legacy: single partition with default precision
|
|
251
|
+
const hasGeohashPartition = resource.config.partitions &&
|
|
252
|
+
resource.config.partitions.byGeohash;
|
|
253
|
+
|
|
254
|
+
if (!hasGeohashPartition) {
|
|
255
|
+
updatedConfig.partitions.byGeohash = {
|
|
256
|
+
fields: {
|
|
257
|
+
_geohash: 'string'
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
partitionsCreated++;
|
|
262
|
+
|
|
263
|
+
if (this.verbose) {
|
|
264
|
+
console.log(`[GeoPlugin] Created byGeohash partition for "${resource.name}"`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// Update resource config
|
|
270
|
+
if (partitionsCreated > 0) {
|
|
271
|
+
resource.config = updatedConfig;
|
|
272
|
+
|
|
273
|
+
// Re-setup partition hooks to register hooks for new geo partitions
|
|
274
|
+
resource.setupPartitionHooks();
|
|
275
|
+
|
|
276
|
+
// Persist to metadata
|
|
277
|
+
if (this.database.uploadMetadataFile) {
|
|
278
|
+
await this.database.uploadMetadataFile();
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Add hooks to automatically calculate geohash at all zoom levels
|
|
285
|
+
*/
|
|
286
|
+
_addHooks(resource, config) {
|
|
287
|
+
const calculateGeohash = async (data) => {
|
|
288
|
+
const lat = data[config.latField];
|
|
289
|
+
const lon = data[config.lonField];
|
|
290
|
+
|
|
291
|
+
if (lat !== undefined && lon !== undefined) {
|
|
292
|
+
// Calculate geohash at default precision
|
|
293
|
+
const geohash = this.encodeGeohash(lat, lon, config.precision);
|
|
294
|
+
|
|
295
|
+
if (config.addGeohash) {
|
|
296
|
+
data.geohash = geohash;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// Always set _geohash for partition support
|
|
300
|
+
data._geohash = geohash;
|
|
301
|
+
|
|
302
|
+
// If zoomLevels configured, calculate geohash for each zoom level
|
|
303
|
+
if (config.zoomLevels && Array.isArray(config.zoomLevels)) {
|
|
304
|
+
for (const zoom of config.zoomLevels) {
|
|
305
|
+
const zoomGeohash = this.encodeGeohash(lat, lon, zoom);
|
|
306
|
+
data[`_geohash_zoom${zoom}`] = zoomGeohash;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
return data;
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
resource.addHook('beforeInsert', calculateGeohash);
|
|
315
|
+
resource.addHook('beforeUpdate', calculateGeohash);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Add helper methods to resource
|
|
320
|
+
*/
|
|
321
|
+
_addHelperMethods(resource, config) {
|
|
322
|
+
const plugin = this;
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Find nearby locations within radius
|
|
326
|
+
* Automatically selects optimal zoom level if multi-zoom enabled
|
|
327
|
+
*/
|
|
328
|
+
resource.findNearby = async function({ lat, lon, lng, radius = 10, limit = 100 }) {
|
|
329
|
+
// Support both 'lon' and 'lng' for backward compatibility
|
|
330
|
+
const longitude = lon !== undefined ? lon : lng;
|
|
331
|
+
|
|
332
|
+
if (lat === undefined || longitude === undefined) {
|
|
333
|
+
throw new Error('lat and lon are required for findNearby');
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
let allRecords = [];
|
|
337
|
+
|
|
338
|
+
// Use partitions if enabled for efficient queries
|
|
339
|
+
if (config.usePartitions) {
|
|
340
|
+
let partitionName, fieldName, precision;
|
|
341
|
+
|
|
342
|
+
// Select optimal zoom if multi-zoom configured
|
|
343
|
+
if (config.zoomLevels && config.zoomLevels.length > 0) {
|
|
344
|
+
const optimalZoom = plugin._selectOptimalZoom(config.zoomLevels, radius);
|
|
345
|
+
partitionName = `byGeohashZoom${optimalZoom}`;
|
|
346
|
+
fieldName = `_geohash_zoom${optimalZoom}`;
|
|
347
|
+
precision = optimalZoom;
|
|
348
|
+
|
|
349
|
+
if (plugin.verbose) {
|
|
350
|
+
console.log(
|
|
351
|
+
`[GeoPlugin] Auto-selected zoom${optimalZoom} (${plugin._getPrecisionDistance(optimalZoom)}km cells) ` +
|
|
352
|
+
`for ${radius}km radius query`
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
} else {
|
|
356
|
+
// Legacy single partition
|
|
357
|
+
partitionName = 'byGeohash';
|
|
358
|
+
fieldName = '_geohash';
|
|
359
|
+
precision = config.precision;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// Check if partition exists
|
|
363
|
+
if (this.config.partitions?.[partitionName]) {
|
|
364
|
+
// Calculate center geohash at selected precision
|
|
365
|
+
const centerGeohash = plugin.encodeGeohash(lat, longitude, precision);
|
|
366
|
+
|
|
367
|
+
// Get neighboring geohashes to cover the search area
|
|
368
|
+
const neighbors = plugin.getNeighbors(centerGeohash);
|
|
369
|
+
const geohashesToSearch = [centerGeohash, ...neighbors];
|
|
370
|
+
|
|
371
|
+
// Query each geohash partition in parallel
|
|
372
|
+
const partitionResults = await Promise.all(
|
|
373
|
+
geohashesToSearch.map(async (geohash) => {
|
|
374
|
+
const [ok, err, records] = await tryFn(async () => {
|
|
375
|
+
return await this.listPartition({
|
|
376
|
+
partition: partitionName,
|
|
377
|
+
partitionValues: { [fieldName]: geohash },
|
|
378
|
+
limit: limit * 2
|
|
379
|
+
});
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
return ok ? records : [];
|
|
383
|
+
})
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
// Flatten results
|
|
387
|
+
allRecords = partitionResults.flat();
|
|
388
|
+
|
|
389
|
+
if (plugin.verbose) {
|
|
390
|
+
console.log(
|
|
391
|
+
`[GeoPlugin] findNearby searched ${geohashesToSearch.length} ${partitionName} partitions, ` +
|
|
392
|
+
`found ${allRecords.length} candidates`
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
} else {
|
|
396
|
+
// Fallback to full scan if partition doesn't exist
|
|
397
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
398
|
+
}
|
|
399
|
+
} else {
|
|
400
|
+
// Fallback to full scan if partitions not enabled
|
|
401
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// Calculate distances and filter
|
|
405
|
+
const withDistances = allRecords
|
|
406
|
+
.map(record => {
|
|
407
|
+
const recordLat = record[config.latField];
|
|
408
|
+
const recordLon = record[config.lonField];
|
|
409
|
+
|
|
410
|
+
if (recordLat === undefined || recordLon === undefined) {
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
const distance = plugin.calculateDistance(lat, longitude, recordLat, recordLon);
|
|
415
|
+
|
|
416
|
+
return {
|
|
417
|
+
...record,
|
|
418
|
+
_distance: distance
|
|
419
|
+
};
|
|
420
|
+
})
|
|
421
|
+
.filter(record => record !== null && record._distance <= radius)
|
|
422
|
+
.sort((a, b) => a._distance - b._distance)
|
|
423
|
+
.slice(0, limit);
|
|
424
|
+
|
|
425
|
+
return withDistances;
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Find locations within bounding box
|
|
430
|
+
* Automatically selects optimal zoom level if multi-zoom enabled
|
|
431
|
+
*/
|
|
432
|
+
resource.findInBounds = async function({ north, south, east, west, limit = 100 }) {
|
|
433
|
+
if (north === undefined || south === undefined || east === undefined || west === undefined) {
|
|
434
|
+
throw new Error('north, south, east, west are required for findInBounds');
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
let allRecords = [];
|
|
438
|
+
|
|
439
|
+
// Use partitions if enabled for efficient queries
|
|
440
|
+
if (config.usePartitions) {
|
|
441
|
+
let partitionName, precision;
|
|
442
|
+
|
|
443
|
+
// Select optimal zoom if multi-zoom configured
|
|
444
|
+
if (config.zoomLevels && config.zoomLevels.length > 0) {
|
|
445
|
+
// Calculate approximate diameter of bounding box for zoom selection
|
|
446
|
+
const centerLat = (north + south) / 2;
|
|
447
|
+
const centerLon = (east + west) / 2;
|
|
448
|
+
const latRadius = plugin.calculateDistance(centerLat, centerLon, north, centerLon);
|
|
449
|
+
const lonRadius = plugin.calculateDistance(centerLat, centerLon, centerLat, east);
|
|
450
|
+
const approximateRadius = Math.max(latRadius, lonRadius);
|
|
451
|
+
|
|
452
|
+
const optimalZoom = plugin._selectOptimalZoom(config.zoomLevels, approximateRadius);
|
|
453
|
+
partitionName = `byGeohashZoom${optimalZoom}`;
|
|
454
|
+
precision = optimalZoom;
|
|
455
|
+
|
|
456
|
+
if (plugin.verbose) {
|
|
457
|
+
console.log(
|
|
458
|
+
`[GeoPlugin] Auto-selected zoom${optimalZoom} (${plugin._getPrecisionDistance(optimalZoom)}km cells) ` +
|
|
459
|
+
`for ${approximateRadius.toFixed(1)}km bounding box`
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
} else {
|
|
463
|
+
// Legacy single partition
|
|
464
|
+
partitionName = 'byGeohash';
|
|
465
|
+
precision = config.precision;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// Check if partition exists
|
|
469
|
+
if (this.config.partitions?.[partitionName]) {
|
|
470
|
+
// Calculate all geohashes that cover the bounding box
|
|
471
|
+
const geohashesToSearch = plugin._getGeohashesInBounds({
|
|
472
|
+
north, south, east, west,
|
|
473
|
+
precision
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
// Query each geohash partition in parallel
|
|
477
|
+
const partitionResults = await Promise.all(
|
|
478
|
+
geohashesToSearch.map(async (geohash) => {
|
|
479
|
+
const [ok, err, records] = await tryFn(async () => {
|
|
480
|
+
const fieldName = config.zoomLevels ? `_geohash_zoom${precision}` : '_geohash';
|
|
481
|
+
return await this.listPartition({
|
|
482
|
+
partition: partitionName,
|
|
483
|
+
partitionValues: { [fieldName]: geohash },
|
|
484
|
+
limit: limit * 2
|
|
485
|
+
});
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
return ok ? records : [];
|
|
489
|
+
})
|
|
490
|
+
);
|
|
491
|
+
|
|
492
|
+
// Flatten results
|
|
493
|
+
allRecords = partitionResults.flat();
|
|
494
|
+
|
|
495
|
+
if (plugin.verbose) {
|
|
496
|
+
console.log(
|
|
497
|
+
`[GeoPlugin] findInBounds searched ${geohashesToSearch.length} ${partitionName} partitions, ` +
|
|
498
|
+
`found ${allRecords.length} candidates`
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
} else {
|
|
502
|
+
// Fallback to full scan if partition doesn't exist
|
|
503
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
504
|
+
}
|
|
505
|
+
} else {
|
|
506
|
+
// Fallback to full scan if partitions not enabled
|
|
507
|
+
allRecords = await this.list({ limit: limit * 10 });
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// Filter by exact bounding box (geohash cells may extend beyond bounds)
|
|
511
|
+
const inBounds = allRecords
|
|
512
|
+
.filter(record => {
|
|
513
|
+
const lat = record[config.latField];
|
|
514
|
+
const lon = record[config.lonField];
|
|
515
|
+
|
|
516
|
+
if (lat === undefined || lon === undefined) {
|
|
517
|
+
return false;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
return lat <= north && lat >= south && lon <= east && lon >= west;
|
|
521
|
+
})
|
|
522
|
+
.slice(0, limit);
|
|
523
|
+
|
|
524
|
+
return inBounds;
|
|
525
|
+
};
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Get distance between two records
|
|
529
|
+
*/
|
|
530
|
+
resource.getDistance = async function(id1, id2) {
|
|
531
|
+
let record1, record2;
|
|
532
|
+
|
|
533
|
+
try {
|
|
534
|
+
[record1, record2] = await Promise.all([
|
|
535
|
+
this.get(id1),
|
|
536
|
+
this.get(id2)
|
|
537
|
+
]);
|
|
538
|
+
} catch (err) {
|
|
539
|
+
if (err.name === 'NoSuchKey' || err.message?.includes('No such key')) {
|
|
540
|
+
throw new Error('One or both records not found');
|
|
541
|
+
}
|
|
542
|
+
throw err;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
if (!record1 || !record2) {
|
|
546
|
+
throw new Error('One or both records not found');
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
const lat1 = record1[config.latField];
|
|
550
|
+
const lon1 = record1[config.lonField];
|
|
551
|
+
const lat2 = record2[config.latField];
|
|
552
|
+
const lon2 = record2[config.lonField];
|
|
553
|
+
|
|
554
|
+
if (lat1 === undefined || lon1 === undefined || lat2 === undefined || lon2 === undefined) {
|
|
555
|
+
throw new Error('One or both records missing coordinates');
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
const distance = plugin.calculateDistance(lat1, lon1, lat2, lon2);
|
|
559
|
+
|
|
560
|
+
return {
|
|
561
|
+
distance,
|
|
562
|
+
unit: 'km',
|
|
563
|
+
from: id1,
|
|
564
|
+
to: id2
|
|
565
|
+
};
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* Encode coordinates to geohash
|
|
571
|
+
* @param {number} latitude - Latitude (-90 to 90)
|
|
572
|
+
* @param {number} longitude - Longitude (-180 to 180)
|
|
573
|
+
* @param {number} precision - Number of characters in geohash
|
|
574
|
+
* @returns {string} Geohash string
|
|
575
|
+
*/
|
|
576
|
+
encodeGeohash(latitude, longitude, precision = 5) {
|
|
577
|
+
let idx = 0;
|
|
578
|
+
let bit = 0;
|
|
579
|
+
let evenBit = true;
|
|
580
|
+
let geohash = '';
|
|
581
|
+
|
|
582
|
+
let latMin = -90;
|
|
583
|
+
let latMax = 90;
|
|
584
|
+
let lonMin = -180;
|
|
585
|
+
let lonMax = 180;
|
|
586
|
+
|
|
587
|
+
while (geohash.length < precision) {
|
|
588
|
+
if (evenBit) {
|
|
589
|
+
// Longitude
|
|
590
|
+
const lonMid = (lonMin + lonMax) / 2;
|
|
591
|
+
if (longitude > lonMid) {
|
|
592
|
+
idx |= (1 << (4 - bit));
|
|
593
|
+
lonMin = lonMid;
|
|
594
|
+
} else {
|
|
595
|
+
lonMax = lonMid;
|
|
596
|
+
}
|
|
597
|
+
} else {
|
|
598
|
+
// Latitude
|
|
599
|
+
const latMid = (latMin + latMax) / 2;
|
|
600
|
+
if (latitude > latMid) {
|
|
601
|
+
idx |= (1 << (4 - bit));
|
|
602
|
+
latMin = latMid;
|
|
603
|
+
} else {
|
|
604
|
+
latMax = latMid;
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
evenBit = !evenBit;
|
|
609
|
+
|
|
610
|
+
if (bit < 4) {
|
|
611
|
+
bit++;
|
|
612
|
+
} else {
|
|
613
|
+
geohash += this.base32[idx];
|
|
614
|
+
bit = 0;
|
|
615
|
+
idx = 0;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
return geohash;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* Decode geohash to coordinates
|
|
624
|
+
* @param {string} geohash - Geohash string
|
|
625
|
+
* @returns {Object} { latitude, longitude, error }
|
|
626
|
+
*/
|
|
627
|
+
decodeGeohash(geohash) {
|
|
628
|
+
let evenBit = true;
|
|
629
|
+
let latMin = -90;
|
|
630
|
+
let latMax = 90;
|
|
631
|
+
let lonMin = -180;
|
|
632
|
+
let lonMax = 180;
|
|
633
|
+
|
|
634
|
+
for (let i = 0; i < geohash.length; i++) {
|
|
635
|
+
const chr = geohash[i];
|
|
636
|
+
const idx = this.base32.indexOf(chr);
|
|
637
|
+
|
|
638
|
+
if (idx === -1) {
|
|
639
|
+
throw new Error(`Invalid geohash character: ${chr}`);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
for (let n = 4; n >= 0; n--) {
|
|
643
|
+
const bitN = (idx >> n) & 1;
|
|
644
|
+
|
|
645
|
+
if (evenBit) {
|
|
646
|
+
// Longitude
|
|
647
|
+
const lonMid = (lonMin + lonMax) / 2;
|
|
648
|
+
if (bitN === 1) {
|
|
649
|
+
lonMin = lonMid;
|
|
650
|
+
} else {
|
|
651
|
+
lonMax = lonMid;
|
|
652
|
+
}
|
|
653
|
+
} else {
|
|
654
|
+
// Latitude
|
|
655
|
+
const latMid = (latMin + latMax) / 2;
|
|
656
|
+
if (bitN === 1) {
|
|
657
|
+
latMin = latMid;
|
|
658
|
+
} else {
|
|
659
|
+
latMax = latMid;
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
evenBit = !evenBit;
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
const latitude = (latMin + latMax) / 2;
|
|
668
|
+
const longitude = (lonMin + lonMax) / 2;
|
|
669
|
+
|
|
670
|
+
return {
|
|
671
|
+
latitude,
|
|
672
|
+
longitude,
|
|
673
|
+
error: {
|
|
674
|
+
latitude: latMax - latMin,
|
|
675
|
+
longitude: lonMax - lonMin
|
|
676
|
+
}
|
|
677
|
+
};
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
/**
|
|
681
|
+
* Calculate distance between two coordinates using Haversine formula
|
|
682
|
+
* @param {number} lat1 - Latitude of point 1
|
|
683
|
+
* @param {number} lon1 - Longitude of point 1
|
|
684
|
+
* @param {number} lat2 - Latitude of point 2
|
|
685
|
+
* @param {number} lon2 - Longitude of point 2
|
|
686
|
+
* @returns {number} Distance in kilometers
|
|
687
|
+
*/
|
|
688
|
+
calculateDistance(lat1, lon1, lat2, lon2) {
|
|
689
|
+
const R = 6371; // Earth's radius in kilometers
|
|
690
|
+
|
|
691
|
+
const dLat = this._toRadians(lat2 - lat1);
|
|
692
|
+
const dLon = this._toRadians(lon2 - lon1);
|
|
693
|
+
|
|
694
|
+
const a =
|
|
695
|
+
Math.sin(dLat / 2) * Math.sin(dLat / 2) +
|
|
696
|
+
Math.cos(this._toRadians(lat1)) *
|
|
697
|
+
Math.cos(this._toRadians(lat2)) *
|
|
698
|
+
Math.sin(dLon / 2) * Math.sin(dLon / 2);
|
|
699
|
+
|
|
700
|
+
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
|
701
|
+
|
|
702
|
+
return R * c;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* Get geohash neighbors (8 surrounding cells)
|
|
707
|
+
* @param {string} geohash - Center geohash
|
|
708
|
+
* @returns {Array<string>} Array of 8 neighboring geohashes
|
|
709
|
+
*/
|
|
710
|
+
getNeighbors(geohash) {
|
|
711
|
+
const decoded = this.decodeGeohash(geohash);
|
|
712
|
+
const { latitude, longitude, error } = decoded;
|
|
713
|
+
|
|
714
|
+
const latStep = error.latitude;
|
|
715
|
+
const lonStep = error.longitude;
|
|
716
|
+
|
|
717
|
+
const neighbors = [];
|
|
718
|
+
|
|
719
|
+
// 8 directions
|
|
720
|
+
const directions = [
|
|
721
|
+
[-latStep, -lonStep], // SW
|
|
722
|
+
[-latStep, 0], // S
|
|
723
|
+
[-latStep, lonStep], // SE
|
|
724
|
+
[0, -lonStep], // W
|
|
725
|
+
[0, lonStep], // E
|
|
726
|
+
[latStep, -lonStep], // NW
|
|
727
|
+
[latStep, 0], // N
|
|
728
|
+
[latStep, lonStep] // NE
|
|
729
|
+
];
|
|
730
|
+
|
|
731
|
+
for (const [latDelta, lonDelta] of directions) {
|
|
732
|
+
const neighborHash = this.encodeGeohash(
|
|
733
|
+
latitude + latDelta,
|
|
734
|
+
longitude + lonDelta,
|
|
735
|
+
geohash.length
|
|
736
|
+
);
|
|
737
|
+
neighbors.push(neighborHash);
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
return neighbors;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* Get all geohashes that cover a bounding box
|
|
745
|
+
* @param {Object} bounds - Bounding box { north, south, east, west, precision }
|
|
746
|
+
* @returns {Array<string>} Array of unique geohashes covering the area
|
|
747
|
+
*/
|
|
748
|
+
_getGeohashesInBounds({ north, south, east, west, precision }) {
|
|
749
|
+
const geohashes = new Set();
|
|
750
|
+
|
|
751
|
+
// Calculate step size based on precision
|
|
752
|
+
const cellSize = this._getPrecisionDistance(precision);
|
|
753
|
+
// Convert km to degrees (rough approximation: 1 degree ≈ 111 km)
|
|
754
|
+
const latStep = cellSize / 111;
|
|
755
|
+
const lonStep = cellSize / (111 * Math.cos(this._toRadians((north + south) / 2)));
|
|
756
|
+
|
|
757
|
+
// Generate grid of points and calculate their geohashes
|
|
758
|
+
for (let lat = south; lat <= north; lat += latStep) {
|
|
759
|
+
for (let lon = west; lon <= east; lon += lonStep) {
|
|
760
|
+
const geohash = this.encodeGeohash(lat, lon, precision);
|
|
761
|
+
geohashes.add(geohash);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
// Also add geohashes for the corners and edges to ensure full coverage
|
|
766
|
+
const corners = [
|
|
767
|
+
[north, west], [north, east],
|
|
768
|
+
[south, west], [south, east],
|
|
769
|
+
[(north + south) / 2, west], [(north + south) / 2, east],
|
|
770
|
+
[north, (east + west) / 2], [south, (east + west) / 2]
|
|
771
|
+
];
|
|
772
|
+
|
|
773
|
+
for (const [lat, lon] of corners) {
|
|
774
|
+
const geohash = this.encodeGeohash(lat, lon, precision);
|
|
775
|
+
geohashes.add(geohash);
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
return Array.from(geohashes);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Convert degrees to radians
|
|
783
|
+
*/
|
|
784
|
+
_toRadians(degrees) {
|
|
785
|
+
return degrees * (Math.PI / 180);
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Get approximate cell size for precision level
|
|
790
|
+
*/
|
|
791
|
+
_getPrecisionDistance(precision) {
|
|
792
|
+
const distances = {
|
|
793
|
+
1: 5000,
|
|
794
|
+
2: 1250,
|
|
795
|
+
3: 156,
|
|
796
|
+
4: 39,
|
|
797
|
+
5: 4.9,
|
|
798
|
+
6: 1.2,
|
|
799
|
+
7: 0.15,
|
|
800
|
+
8: 0.038,
|
|
801
|
+
9: 0.0047,
|
|
802
|
+
10: 0.0012,
|
|
803
|
+
11: 0.00015,
|
|
804
|
+
12: 0.000037
|
|
805
|
+
};
|
|
806
|
+
|
|
807
|
+
return distances[precision] || 5;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* Select optimal zoom level based on search radius
|
|
812
|
+
* @param {Array<number>} zoomLevels - Available zoom levels
|
|
813
|
+
* @param {number} radiusKm - Search radius in kilometers
|
|
814
|
+
* @returns {number} Optimal zoom precision
|
|
815
|
+
*/
|
|
816
|
+
_selectOptimalZoom(zoomLevels, radiusKm) {
|
|
817
|
+
if (!zoomLevels || zoomLevels.length === 0) {
|
|
818
|
+
return null;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// Select zoom where cell size is approximately 2-3x smaller than radius
|
|
822
|
+
// This gives good coverage without too many partitions to query
|
|
823
|
+
const targetCellSize = radiusKm / 2.5;
|
|
824
|
+
|
|
825
|
+
let bestZoom = zoomLevels[0];
|
|
826
|
+
let bestDiff = Math.abs(this._getPrecisionDistance(bestZoom) - targetCellSize);
|
|
827
|
+
|
|
828
|
+
for (const zoom of zoomLevels) {
|
|
829
|
+
const cellSize = this._getPrecisionDistance(zoom);
|
|
830
|
+
const diff = Math.abs(cellSize - targetCellSize);
|
|
831
|
+
|
|
832
|
+
if (diff < bestDiff) {
|
|
833
|
+
bestDiff = diff;
|
|
834
|
+
bestZoom = zoom;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
return bestZoom;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/**
|
|
842
|
+
* Get plugin statistics
|
|
843
|
+
*/
|
|
844
|
+
getStats() {
|
|
845
|
+
return {
|
|
846
|
+
resources: Object.keys(this.resources).length,
|
|
847
|
+
configurations: Object.entries(this.resources).map(([name, config]) => ({
|
|
848
|
+
resource: name,
|
|
849
|
+
latField: config.latField,
|
|
850
|
+
lonField: config.lonField,
|
|
851
|
+
precision: config.precision,
|
|
852
|
+
cellSize: `~${this._getPrecisionDistance(config.precision)}km`
|
|
853
|
+
}))
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
/**
|
|
858
|
+
* Uninstall the plugin
|
|
859
|
+
*/
|
|
860
|
+
async uninstall() {
|
|
861
|
+
if (this.verbose) {
|
|
862
|
+
console.log('[GeoPlugin] Uninstalled');
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
this.emit('uninstalled', {
|
|
866
|
+
plugin: 'GeoPlugin'
|
|
867
|
+
});
|
|
868
|
+
|
|
869
|
+
await super.uninstall();
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
export default GeoPlugin;
|