leadintel 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leadintel/__init__.py +6 -0
- leadintel/__main__.py +6 -0
- leadintel/config.py +165 -0
- leadintel/data/__init__.py +1 -0
- leadintel/data/demo.py +315 -0
- leadintel/exporters/__init__.py +6 -0
- leadintel/exporters/csv_exporter.py +113 -0
- leadintel/exporters/excel_exporter.py +123 -0
- leadintel/models/__init__.py +6 -0
- leadintel/models/business.py +203 -0
- leadintel/models/report.py +31 -0
- leadintel/providers/__init__.py +51 -0
- leadintel/providers/base.py +54 -0
- leadintel/providers/demo_provider.py +65 -0
- leadintel/providers/overpass_provider.py +1040 -0
- leadintel/providers/registry.py +151 -0
- leadintel/server.py +61 -0
- leadintel/services/__init__.py +48 -0
- leadintel/services/ai_business_insights.py +310 -0
- leadintel/services/business_comparison.py +220 -0
- leadintel/services/business_report.py +294 -0
- leadintel/services/competitor_research.py +293 -0
- leadintel/services/lead_scoring.py +385 -0
- leadintel/services/local_ai.py +186 -0
- leadintel/services/robots_checker.py +81 -0
- leadintel/services/seo_auditor.py +281 -0
- leadintel/services/social_finder.py +119 -0
- leadintel/services/url_validator.py +295 -0
- leadintel/services/website_analyzer.py +213 -0
- leadintel/services/website_opportunity.py +523 -0
- leadintel/storage/__init__.py +12 -0
- leadintel/storage/db.py +189 -0
- leadintel/storage/path_validator.py +118 -0
- leadintel/tools/__init__.py +38 -0
- leadintel/tools/analyze_website.py +57 -0
- leadintel/tools/check_seo.py +64 -0
- leadintel/tools/compare_businesses.py +65 -0
- leadintel/tools/export_leads.py +112 -0
- leadintel/tools/find_business_website.py +66 -0
- leadintel/tools/find_competitors.py +75 -0
- leadintel/tools/find_social_profiles.py +85 -0
- leadintel/tools/generate_business_insights.py +93 -0
- leadintel/tools/generate_business_report.py +92 -0
- leadintel/tools/generate_outreach.py +153 -0
- leadintel/tools/get_business_details.py +96 -0
- leadintel/tools/get_saved_leads.py +58 -0
- leadintel/tools/score_lead.py +88 -0
- leadintel/tools/search_businesses.py +305 -0
- leadintel/tools/website_finder.py +147 -0
- leadintel-0.3.0.dist-info/METADATA +351 -0
- leadintel-0.3.0.dist-info/RECORD +54 -0
- leadintel-0.3.0.dist-info/WHEEL +4 -0
- leadintel-0.3.0.dist-info/entry_points.txt +3 -0
- leadintel-0.3.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1040 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OverpassProvider — free business discovery via OpenStreetMap Overpass API.
|
|
3
|
+
|
|
4
|
+
Attribution requirement (ODbL):
|
|
5
|
+
Data © OpenStreetMap contributors, ODbL 1.0.
|
|
6
|
+
https://www.openstreetmap.org/copyright
|
|
7
|
+
|
|
8
|
+
Usage policy:
|
|
9
|
+
- Rate-limit: ≤1 req/s; do not hammer the public endpoint.
|
|
10
|
+
- Use the public endpoint: https://overpass-api.de/api/interpreter
|
|
11
|
+
or a self-hosted instance (set OVERPASS_ENDPOINT env var).
|
|
12
|
+
- Never cache results longer than 7 days to honour OSM data freshness.
|
|
13
|
+
- Always display the ODbL attribution string in exports.
|
|
14
|
+
|
|
15
|
+
This provider:
|
|
16
|
+
* Geocodes location strings via Nominatim (also free, ODbL).
|
|
17
|
+
* Queries Overpass QL for named businesses in a bounding box.
|
|
18
|
+
* Maps OSM tags → Business model fields.
|
|
19
|
+
* Implements a simple in-process TTL cache to avoid hammering the API.
|
|
20
|
+
* Falls back to Nominatim POI search when Overpass is at capacity.
|
|
21
|
+
* Enforces a global 1-second Nominatim rate-limit across all instances.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import asyncio
|
|
27
|
+
import hashlib
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
import math
|
|
31
|
+
import time
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
import httpx
|
|
35
|
+
|
|
36
|
+
from ..models.business import Business
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
# ── Constants ─────────────────────────────────────────────────────────────────
|
|
41
|
+
OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter"
|
|
42
|
+
NOMINATIM_ENDPOINT = "https://nominatim.openstreetmap.org/search"
|
|
43
|
+
NOMINATIM_REVERSE_ENDPOINT = "https://nominatim.openstreetmap.org/reverse"
|
|
44
|
+
NOMINATIM_LOOKUP_ENDPOINT = "https://nominatim.openstreetmap.org/lookup"
|
|
45
|
+
OSM_ATTRIBUTION = "Data © OpenStreetMap contributors, ODbL 1.0"
|
|
46
|
+
# Descriptive User-Agent per Nominatim usage policy: https://operations.osmfoundation.org/policies/nominatim/
|
|
47
|
+
USER_AGENT = "LeadIntel/0.2.0 (open-source B2B lead-generation MCP server; https://github.com/example/leadintel)"
|
|
48
|
+
|
|
49
|
+
OVERPASS_FALLBACK_ENDPOINTS = [
|
|
50
|
+
"https://lz4.overpass-api.de/api/interpreter",
|
|
51
|
+
"https://z.overpass-api.de/api/interpreter",
|
|
52
|
+
"https://overpass.private.coffee/api/interpreter",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
# Default search radius around a geocoded point, in metres (6km covers central urban core)
|
|
56
|
+
DEFAULT_RADIUS_M = 6_000
|
|
57
|
+
# Max radius to keep queries fast
|
|
58
|
+
MAX_RADIUS_M = 50_000
|
|
59
|
+
# Overpass request timeout (seconds)
|
|
60
|
+
REQUEST_TIMEOUT = 30.0
|
|
61
|
+
# Minimum seconds between consecutive Overpass requests (rate-limit)
|
|
62
|
+
MIN_REQUEST_INTERVAL = 1.1
|
|
63
|
+
# Nominatim usage policy: https://operations.osmfoundation.org/policies/nominatim/
|
|
64
|
+
# Minimum 1-second interval between any Nominatim requests (global across all instances)
|
|
65
|
+
NOMINATIM_MIN_INTERVAL = 1.0
|
|
66
|
+
# Simple in-process cache TTL (seconds); never exceed 7 days (604 800 s)
|
|
67
|
+
CACHE_TTL = 3_600 # 1 hour
|
|
68
|
+
# Nominatim geocode cache TTL — city name → coords; reuse to avoid repeat requests
|
|
69
|
+
NOMINATIM_GEOCODE_TTL = 86_400 # 24 hours; city locations do not change
|
|
70
|
+
|
|
71
|
+
# Global Nominatim rate-limit clock — shared across all OverpassProvider instances
|
|
72
|
+
# so concurrent searches in the same process cannot exceed the usage policy limit.
|
|
73
|
+
_nominatim_last_request: float = 0.0
|
|
74
|
+
_nominatim_lock: asyncio.Lock | None = None # initialised lazily; must be in an event loop
|
|
75
|
+
|
|
76
|
+
# Global Nominatim geocode cache: location string → (coords (lat, lon), timestamp)
|
|
77
|
+
_nominatim_geocode_cache: dict[str, tuple[tuple[float, float], float]] = {}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def haversine_distance_meters(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
|
81
|
+
"""Calculate the great-circle distance between two points on Earth in metres."""
|
|
82
|
+
r = 6_371_000.0 # Earth radius in metres
|
|
83
|
+
phi1 = math.radians(lat1)
|
|
84
|
+
phi2 = math.radians(lat2)
|
|
85
|
+
delta_phi = math.radians(lat2 - lat1)
|
|
86
|
+
delta_lambda = math.radians(lon2 - lon1)
|
|
87
|
+
|
|
88
|
+
a = (
|
|
89
|
+
math.sin(delta_phi / 2.0) ** 2
|
|
90
|
+
+ math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2.0) ** 2
|
|
91
|
+
)
|
|
92
|
+
c = 2.0 * math.atan2(math.sqrt(a), math.sqrt(1.0 - a))
|
|
93
|
+
return r * c
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def _nominatim_rate_limit() -> None:
|
|
97
|
+
"""
|
|
98
|
+
Enforce a global minimum 1-second gap between all Nominatim HTTP requests.
|
|
99
|
+
|
|
100
|
+
This is shared across every OverpassProvider instance in the same process so
|
|
101
|
+
concurrent tool calls never violate the Nominatim Acceptable Use Policy
|
|
102
|
+
(https://operations.osmfoundation.org/policies/nominatim/).
|
|
103
|
+
|
|
104
|
+
Uses a module-level asyncio.Lock (initialised lazily on first call so it is
|
|
105
|
+
always created inside a running event loop) to serialise concurrent callers.
|
|
106
|
+
"""
|
|
107
|
+
global _nominatim_last_request, _nominatim_lock # noqa: PLW0603
|
|
108
|
+
if _nominatim_lock is None:
|
|
109
|
+
_nominatim_lock = asyncio.Lock()
|
|
110
|
+
async with _nominatim_lock:
|
|
111
|
+
elapsed = time.monotonic() - _nominatim_last_request
|
|
112
|
+
if elapsed < NOMINATIM_MIN_INTERVAL:
|
|
113
|
+
await asyncio.sleep(NOMINATIM_MIN_INTERVAL - elapsed)
|
|
114
|
+
_nominatim_last_request = time.monotonic()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ── OSM tag → category mapping ────────────────────────────────────────────────
|
|
118
|
+
_OSM_AMENITY_TO_CATEGORY: dict[str, str] = {
|
|
119
|
+
"restaurant": "Food & Beverage",
|
|
120
|
+
"cafe": "Food & Beverage",
|
|
121
|
+
"bar": "Food & Beverage",
|
|
122
|
+
"pub": "Food & Beverage",
|
|
123
|
+
"fast_food": "Food & Beverage",
|
|
124
|
+
"hospital": "Healthcare",
|
|
125
|
+
"clinic": "Healthcare",
|
|
126
|
+
"pharmacy": "Healthcare",
|
|
127
|
+
"dentist": "Healthcare",
|
|
128
|
+
"school": "Education",
|
|
129
|
+
"university": "Education",
|
|
130
|
+
"college": "Education",
|
|
131
|
+
"bank": "Financial Services",
|
|
132
|
+
"atm": "Financial Services",
|
|
133
|
+
"hotel": "Hospitality",
|
|
134
|
+
"hostel": "Hospitality",
|
|
135
|
+
"guest_house": "Hospitality",
|
|
136
|
+
"gym": "Health & Fitness",
|
|
137
|
+
"fuel": "Automotive",
|
|
138
|
+
"car_wash": "Automotive",
|
|
139
|
+
"cinema": "Entertainment",
|
|
140
|
+
"theatre": "Entertainment",
|
|
141
|
+
"library": "Education",
|
|
142
|
+
"post_office": "Government Services",
|
|
143
|
+
"police": "Government Services",
|
|
144
|
+
"fire_station": "Government Services",
|
|
145
|
+
"place_of_worship": "Religion",
|
|
146
|
+
"marketplace": "Retail",
|
|
147
|
+
"supermarket": "Retail",
|
|
148
|
+
"convenience": "Retail",
|
|
149
|
+
"hairdresser": "Personal Care",
|
|
150
|
+
"beauty": "Personal Care",
|
|
151
|
+
"laundry": "Personal Care",
|
|
152
|
+
"shop": "Retail",
|
|
153
|
+
"office": "Professional Services",
|
|
154
|
+
"coworking_space": "Professional Services",
|
|
155
|
+
"it": "Technology",
|
|
156
|
+
"accountant": "Professional Services",
|
|
157
|
+
"lawyer": "Legal Services",
|
|
158
|
+
"estate_agent": "Real Estate",
|
|
159
|
+
"travel_agency": "Travel",
|
|
160
|
+
"car_rental": "Automotive",
|
|
161
|
+
"charging_station": "Automotive",
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
_OSM_SHOP_TO_CATEGORY: dict[str, str] = {
|
|
165
|
+
"supermarket": "Retail – Grocery",
|
|
166
|
+
"clothes": "Retail – Fashion",
|
|
167
|
+
"electronics": "Retail – Electronics",
|
|
168
|
+
"furniture": "Retail – Furniture",
|
|
169
|
+
"hardware": "Retail – Hardware",
|
|
170
|
+
"bakery": "Food & Beverage",
|
|
171
|
+
"butcher": "Food & Beverage",
|
|
172
|
+
"florist": "Retail – Florist",
|
|
173
|
+
"books": "Retail – Books",
|
|
174
|
+
"sports": "Retail – Sports",
|
|
175
|
+
"jewelry": "Retail – Jewellery",
|
|
176
|
+
"computer": "Technology",
|
|
177
|
+
"mobile_phone": "Technology",
|
|
178
|
+
"car": "Automotive – Dealer",
|
|
179
|
+
"bicycle": "Retail – Bicycle",
|
|
180
|
+
"optician": "Healthcare – Optician",
|
|
181
|
+
"chemist": "Healthcare – Pharmacy",
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
_OSM_OFFICE_TO_CATEGORY: dict[str, str] = {
|
|
185
|
+
"it": "Technology",
|
|
186
|
+
"software": "Technology",
|
|
187
|
+
"company": "Professional Services",
|
|
188
|
+
"consulting": "Consulting",
|
|
189
|
+
"lawyer": "Legal Services",
|
|
190
|
+
"accountant": "Accounting",
|
|
191
|
+
"architect": "Architecture",
|
|
192
|
+
"engineer": "Engineering",
|
|
193
|
+
"financial": "Financial Services",
|
|
194
|
+
"insurance": "Insurance",
|
|
195
|
+
"marketing": "Marketing",
|
|
196
|
+
"advertising": "Marketing",
|
|
197
|
+
"real_estate": "Real Estate",
|
|
198
|
+
"travel_agent": "Travel",
|
|
199
|
+
"government": "Government",
|
|
200
|
+
"ngo": "Non-Profit",
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _osm_tags_to_category(tags: dict[str, str]) -> str | None:
|
|
205
|
+
amenity = tags.get("amenity", "")
|
|
206
|
+
shop = tags.get("shop", "")
|
|
207
|
+
office = tags.get("office", "")
|
|
208
|
+
|
|
209
|
+
if amenity:
|
|
210
|
+
return _OSM_AMENITY_TO_CATEGORY.get(amenity, amenity.replace("_", " ").title())
|
|
211
|
+
if shop:
|
|
212
|
+
return _OSM_SHOP_TO_CATEGORY.get(shop, shop.replace("_", " ").title())
|
|
213
|
+
if office:
|
|
214
|
+
return _OSM_OFFICE_TO_CATEGORY.get(office, office.replace("_", " ").title())
|
|
215
|
+
|
|
216
|
+
craft = tags.get("craft", "")
|
|
217
|
+
if craft:
|
|
218
|
+
return craft.replace("_", " ").title()
|
|
219
|
+
|
|
220
|
+
tourism = tags.get("tourism", "")
|
|
221
|
+
if tourism:
|
|
222
|
+
return tourism.replace("_", " ").title()
|
|
223
|
+
|
|
224
|
+
healthcare = tags.get("healthcare", "")
|
|
225
|
+
if healthcare:
|
|
226
|
+
return "Healthcare"
|
|
227
|
+
|
|
228
|
+
leisure = tags.get("leisure", "")
|
|
229
|
+
if leisure:
|
|
230
|
+
return leisure.replace("_", " ").title()
|
|
231
|
+
|
|
232
|
+
return None
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def is_valid_business_element(element: dict[str, Any]) -> bool:
|
|
236
|
+
"""
|
|
237
|
+
Validate whether an OSM element is an actual business or commercial POI.
|
|
238
|
+
Strictly filters out roads, ways, railways, natural boundaries, bus stops, and infrastructure.
|
|
239
|
+
"""
|
|
240
|
+
tags: dict[str, str] = element.get("tags", {})
|
|
241
|
+
name = tags.get("name") or tags.get("brand") or tags.get("operator")
|
|
242
|
+
if not name:
|
|
243
|
+
return False
|
|
244
|
+
|
|
245
|
+
# Commercial classification tags
|
|
246
|
+
has_commercial_tag = any(
|
|
247
|
+
k in tags
|
|
248
|
+
for k in (
|
|
249
|
+
"amenity", "shop", "office", "craft", "tourism",
|
|
250
|
+
"leisure", "healthcare", "commercial", "brand", "operator"
|
|
251
|
+
)
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
# Reject road ways and transport infrastructure
|
|
255
|
+
highway = tags.get("highway")
|
|
256
|
+
if highway:
|
|
257
|
+
# A road or street is NOT a business unless it carries a specific business amenity/shop/office
|
|
258
|
+
if not any(k in tags for k in ("shop", "office", "craft")):
|
|
259
|
+
amenity = tags.get("amenity")
|
|
260
|
+
if amenity not in ("restaurant", "cafe", "fast_food", "fuel", "bar", "pub", "pharmacy", "bank"):
|
|
261
|
+
return False
|
|
262
|
+
|
|
263
|
+
# Reject non-business infrastructure
|
|
264
|
+
if any(k in tags for k in ("railway", "waterway", "natural", "boundary", "barrier", "traffic_calming")):
|
|
265
|
+
if not has_commercial_tag:
|
|
266
|
+
return False
|
|
267
|
+
|
|
268
|
+
# Reject geographic boundaries, suburbs, or cities
|
|
269
|
+
if "place" in tags and not has_commercial_tag:
|
|
270
|
+
return False
|
|
271
|
+
|
|
272
|
+
return True
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _osm_element_to_business(element: dict[str, Any]) -> Business | None:
|
|
276
|
+
"""Convert a raw Overpass API element to a Business. Returns None if unusable or non-business."""
|
|
277
|
+
if not is_valid_business_element(element):
|
|
278
|
+
return None
|
|
279
|
+
|
|
280
|
+
tags: dict[str, str] = element.get("tags", {})
|
|
281
|
+
|
|
282
|
+
name = tags.get("name") or tags.get("brand") or tags.get("operator")
|
|
283
|
+
if not name:
|
|
284
|
+
return None
|
|
285
|
+
|
|
286
|
+
# Coordinates — node has lat/lon directly; way/relation use centre
|
|
287
|
+
if element["type"] == "node":
|
|
288
|
+
lat = element.get("lat")
|
|
289
|
+
lon = element.get("lon")
|
|
290
|
+
else:
|
|
291
|
+
centre = element.get("center", {})
|
|
292
|
+
lat = centre.get("lat")
|
|
293
|
+
lon = centre.get("lon")
|
|
294
|
+
|
|
295
|
+
osm_id = f"osm_{element['type']}_{element['id']}"
|
|
296
|
+
|
|
297
|
+
# Address
|
|
298
|
+
addr_parts = [
|
|
299
|
+
tags.get("addr:housenumber", ""),
|
|
300
|
+
tags.get("addr:street", ""),
|
|
301
|
+
]
|
|
302
|
+
street = " ".join(p for p in addr_parts if p).strip()
|
|
303
|
+
city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village")
|
|
304
|
+
state = tags.get("addr:state") or tags.get("addr:province")
|
|
305
|
+
country = tags.get("addr:country")
|
|
306
|
+
postal_code = tags.get("addr:postcode")
|
|
307
|
+
|
|
308
|
+
address_parts = [p for p in [street, city, state, postal_code, country] if p]
|
|
309
|
+
address = ", ".join(address_parts) if address_parts else None
|
|
310
|
+
|
|
311
|
+
return Business(
|
|
312
|
+
id=osm_id,
|
|
313
|
+
name=name,
|
|
314
|
+
category=_osm_tags_to_category(tags),
|
|
315
|
+
address=address,
|
|
316
|
+
city=city,
|
|
317
|
+
state=state,
|
|
318
|
+
country=country,
|
|
319
|
+
postal_code=postal_code,
|
|
320
|
+
latitude=lat,
|
|
321
|
+
longitude=lon,
|
|
322
|
+
phone=tags.get("phone") or tags.get("contact:phone"),
|
|
323
|
+
email=tags.get("email") or tags.get("contact:email"),
|
|
324
|
+
website=tags.get("website") or tags.get("contact:website") or tags.get("url"),
|
|
325
|
+
source="openstreetmap",
|
|
326
|
+
is_ai_generated=False,
|
|
327
|
+
osm_tags=tags,
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
_CATEGORY_TO_TERMS: dict[str, str] = {
|
|
332
|
+
"food & beverage": "restaurants",
|
|
333
|
+
"healthcare": "hospitals",
|
|
334
|
+
"education": "schools",
|
|
335
|
+
"financial services": "banks",
|
|
336
|
+
"hospitality": "hotels",
|
|
337
|
+
"health & fitness": "fitness",
|
|
338
|
+
"automotive": "car",
|
|
339
|
+
"entertainment": "cinema",
|
|
340
|
+
"retail": "supermarkets",
|
|
341
|
+
"personal care": "beauty",
|
|
342
|
+
"professional services": "offices",
|
|
343
|
+
"technology": "software",
|
|
344
|
+
"legal services": "lawyer",
|
|
345
|
+
"real estate": "real_estate",
|
|
346
|
+
"travel": "travel_agency",
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _nominatim_element_to_business(element: dict[str, Any]) -> Business | None:
|
|
351
|
+
"""Convert a raw Nominatim search/lookup element to a Business. Returns None if unusable."""
|
|
352
|
+
osm_type = element.get("osm_type")
|
|
353
|
+
osm_id = element.get("osm_id")
|
|
354
|
+
if not osm_type or not osm_id:
|
|
355
|
+
return None
|
|
356
|
+
|
|
357
|
+
cat_class = element.get("category") or element.get("class", "")
|
|
358
|
+
if cat_class in ("highway", "boundary", "place", "waterway", "railway", "natural"):
|
|
359
|
+
return None
|
|
360
|
+
|
|
361
|
+
name = element.get("name")
|
|
362
|
+
if not name:
|
|
363
|
+
display_name = element.get("display_name", "")
|
|
364
|
+
name = display_name.split(",")[0].strip() if display_name else None
|
|
365
|
+
if not name:
|
|
366
|
+
return None
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
lat = float(element["lat"]) if "lat" in element else None
|
|
370
|
+
lon = float(element["lon"]) if "lon" in element else None
|
|
371
|
+
except (ValueError, TypeError):
|
|
372
|
+
lat = None
|
|
373
|
+
lon = None
|
|
374
|
+
|
|
375
|
+
addr: dict[str, Any] = element.get("address") or {}
|
|
376
|
+
street = addr.get("road") or addr.get("street")
|
|
377
|
+
city = (
|
|
378
|
+
addr.get("city")
|
|
379
|
+
or addr.get("town")
|
|
380
|
+
or addr.get("village")
|
|
381
|
+
or addr.get("suburb")
|
|
382
|
+
or addr.get("county")
|
|
383
|
+
)
|
|
384
|
+
state = addr.get("state") or addr.get("province")
|
|
385
|
+
country = addr.get("country")
|
|
386
|
+
postal_code = addr.get("postcode")
|
|
387
|
+
|
|
388
|
+
address_parts = [p for p in [street, city, state, postal_code, country] if p]
|
|
389
|
+
formatted_address = ", ".join(address_parts) if address_parts else element.get("display_name")
|
|
390
|
+
|
|
391
|
+
extratags: dict[str, Any] = element.get("extratags") or {}
|
|
392
|
+
phone = (
|
|
393
|
+
extratags.get("phone")
|
|
394
|
+
or extratags.get("contact:phone")
|
|
395
|
+
or element.get("phone")
|
|
396
|
+
)
|
|
397
|
+
email = (
|
|
398
|
+
extratags.get("email")
|
|
399
|
+
or extratags.get("contact:email")
|
|
400
|
+
or element.get("email")
|
|
401
|
+
)
|
|
402
|
+
website = (
|
|
403
|
+
extratags.get("website")
|
|
404
|
+
or extratags.get("contact:website")
|
|
405
|
+
or extratags.get("url")
|
|
406
|
+
or element.get("website")
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
cat_type = element.get("type", "")
|
|
410
|
+
category = None
|
|
411
|
+
if cat_class:
|
|
412
|
+
category = _osm_tags_to_category({cat_class: cat_type})
|
|
413
|
+
if not category and cat_type:
|
|
414
|
+
category = _osm_tags_to_category({"amenity": cat_type, "shop": cat_type, "office": cat_type})
|
|
415
|
+
if not category and cat_type:
|
|
416
|
+
category = cat_type.replace("_", " ").title()
|
|
417
|
+
|
|
418
|
+
leadintel_id = f"osm_{osm_type}_{osm_id}"
|
|
419
|
+
|
|
420
|
+
return Business(
|
|
421
|
+
id=leadintel_id,
|
|
422
|
+
name=name,
|
|
423
|
+
category=category,
|
|
424
|
+
address=formatted_address,
|
|
425
|
+
city=city,
|
|
426
|
+
state=state,
|
|
427
|
+
country=country,
|
|
428
|
+
postal_code=postal_code,
|
|
429
|
+
latitude=lat,
|
|
430
|
+
longitude=lon,
|
|
431
|
+
phone=phone,
|
|
432
|
+
email=email,
|
|
433
|
+
website=website,
|
|
434
|
+
source="openstreetmap",
|
|
435
|
+
is_ai_generated=False,
|
|
436
|
+
osm_tags={"class": cat_class, "type": cat_type, **extratags},
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _cache_key(
|
|
441
|
+
query: str,
|
|
442
|
+
location: str | None,
|
|
443
|
+
category: str | None,
|
|
444
|
+
limit: int,
|
|
445
|
+
lat: float | None = None,
|
|
446
|
+
lon: float | None = None,
|
|
447
|
+
radius: int | None = None,
|
|
448
|
+
) -> str:
|
|
449
|
+
raw = json.dumps(
|
|
450
|
+
{
|
|
451
|
+
"q": query,
|
|
452
|
+
"loc": location,
|
|
453
|
+
"cat": category,
|
|
454
|
+
"lim": limit,
|
|
455
|
+
"lat": lat,
|
|
456
|
+
"lon": lon,
|
|
457
|
+
"rad": radius,
|
|
458
|
+
},
|
|
459
|
+
sort_keys=True,
|
|
460
|
+
)
|
|
461
|
+
return hashlib.md5(raw.encode()).hexdigest() # noqa: S324 — non-crypto use
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
class OverpassProvider:
|
|
465
|
+
"""
|
|
466
|
+
Business discovery via the free Overpass API (OpenStreetMap data).
|
|
467
|
+
|
|
468
|
+
Attributes:
|
|
469
|
+
name: Provider identifier shown in responses.
|
|
470
|
+
requires_auth: False — no API key needed.
|
|
471
|
+
endpoint: Overpass interpreter URL (default: public instance).
|
|
472
|
+
radius_m: Search radius in metres around a geocoded location.
|
|
473
|
+
"""
|
|
474
|
+
|
|
475
|
+
name: str = "openstreetmap"
|
|
476
|
+
requires_auth: bool = False
|
|
477
|
+
|
|
478
|
+
def __init__(
|
|
479
|
+
self,
|
|
480
|
+
endpoint: str = OVERPASS_ENDPOINT,
|
|
481
|
+
radius_m: int = DEFAULT_RADIUS_M,
|
|
482
|
+
cache_ttl: int = CACHE_TTL,
|
|
483
|
+
fallback_endpoints: list[str] | None = None,
|
|
484
|
+
) -> None:
|
|
485
|
+
self.endpoint = endpoint
|
|
486
|
+
self.radius_m = min(radius_m, MAX_RADIUS_M)
|
|
487
|
+
self._cache: dict[str, tuple[float, list[Business]]] = {}
|
|
488
|
+
self._cache_ttl = cache_ttl
|
|
489
|
+
self._last_request: float = 0.0
|
|
490
|
+
self.fallback_endpoints = (
|
|
491
|
+
fallback_endpoints
|
|
492
|
+
if fallback_endpoints is not None
|
|
493
|
+
else list(OVERPASS_FALLBACK_ENDPOINTS)
|
|
494
|
+
)
|
|
495
|
+
self.last_search_center: dict[str, float] | None = None
|
|
496
|
+
self.last_radius_meters: int | None = None
|
|
497
|
+
|
|
498
|
+
# ── Public interface ──────────────────────────────────────────────────────
|
|
499
|
+
|
|
500
|
+
async def search(
|
|
501
|
+
self,
|
|
502
|
+
query: str,
|
|
503
|
+
location: str | None = None,
|
|
504
|
+
category: str | None = None,
|
|
505
|
+
limit: int = 10,
|
|
506
|
+
latitude: float | None = None,
|
|
507
|
+
longitude: float | None = None,
|
|
508
|
+
radius_meters: int | None = None,
|
|
509
|
+
wider_metro: bool = False,
|
|
510
|
+
**kwargs: Any,
|
|
511
|
+
) -> list[Business]:
|
|
512
|
+
"""Search OSM for businesses matching the query near the location."""
|
|
513
|
+
if not location and (latitude is None or longitude is None):
|
|
514
|
+
logger.warning("No location or coordinates provided; skipping OSM search.")
|
|
515
|
+
return []
|
|
516
|
+
|
|
517
|
+
eff_radius = (
|
|
518
|
+
(radius_meters if radius_meters is not None else 25_000)
|
|
519
|
+
if wider_metro
|
|
520
|
+
else (radius_meters if radius_meters is not None else self.radius_m)
|
|
521
|
+
)
|
|
522
|
+
eff_radius = min(eff_radius, MAX_RADIUS_M)
|
|
523
|
+
|
|
524
|
+
center_and_bbox = await self._resolve_center_and_bbox(
|
|
525
|
+
location=location,
|
|
526
|
+
latitude=latitude,
|
|
527
|
+
longitude=longitude,
|
|
528
|
+
radius_m=eff_radius,
|
|
529
|
+
)
|
|
530
|
+
if center_and_bbox is None:
|
|
531
|
+
logger.warning("Could not geocode location '%s'; skipping OSM search.", location)
|
|
532
|
+
return []
|
|
533
|
+
|
|
534
|
+
center, bbox = center_and_bbox
|
|
535
|
+
center_lat, center_lon = center
|
|
536
|
+
self.last_search_center = {"latitude": center_lat, "longitude": center_lon}
|
|
537
|
+
self.last_radius_meters = eff_radius
|
|
538
|
+
|
|
539
|
+
key = _cache_key(query, location, category, limit, center_lat, center_lon, eff_radius)
|
|
540
|
+
cached = self._get_cache(key)
|
|
541
|
+
if cached is not None:
|
|
542
|
+
logger.debug("Cache hit for key %s", key)
|
|
543
|
+
return cached
|
|
544
|
+
|
|
545
|
+
raw_businesses: list[Business] = []
|
|
546
|
+
overpass_err: Exception | None = None
|
|
547
|
+
|
|
548
|
+
# 1. Attempt Overpass query
|
|
549
|
+
try:
|
|
550
|
+
raw_businesses = await self._run_overpass(
|
|
551
|
+
query=query, category=category, bbox=bbox, limit=limit * 2
|
|
552
|
+
)
|
|
553
|
+
except Exception as exc: # noqa: BLE001
|
|
554
|
+
overpass_err = exc
|
|
555
|
+
logger.warning("Overpass search failed (%s); trying Nominatim fallback.", exc)
|
|
556
|
+
|
|
557
|
+
# 2. If Overpass failed or returned no results, fall back to Nominatim live search
|
|
558
|
+
if not raw_businesses:
|
|
559
|
+
try:
|
|
560
|
+
raw_businesses = await self._search_nominatim(
|
|
561
|
+
query=query,
|
|
562
|
+
location=location or f"{center_lat},{center_lon}",
|
|
563
|
+
category=category,
|
|
564
|
+
limit=limit * 2,
|
|
565
|
+
bbox=bbox,
|
|
566
|
+
center=center,
|
|
567
|
+
)
|
|
568
|
+
except Exception as exc: # noqa: BLE001
|
|
569
|
+
logger.warning("Nominatim live business search failed: %s", exc)
|
|
570
|
+
if overpass_err:
|
|
571
|
+
raise RuntimeError(
|
|
572
|
+
f"OpenStreetMap search failed (Overpass: {overpass_err}; Nominatim: {exc})"
|
|
573
|
+
) from overpass_err
|
|
574
|
+
|
|
575
|
+
# 3. Calculate distance and strictly enforce radius
|
|
576
|
+
filtered: list[Business] = []
|
|
577
|
+
for biz in raw_businesses:
|
|
578
|
+
if biz.latitude is not None and biz.longitude is not None:
|
|
579
|
+
dist_m = haversine_distance_meters(center_lat, center_lon, biz.latitude, biz.longitude)
|
|
580
|
+
biz.distance_meters = round(dist_m, 1)
|
|
581
|
+
biz.distance_km = round(dist_m / 1000.0, 2)
|
|
582
|
+
else:
|
|
583
|
+
dist_m = None
|
|
584
|
+
|
|
585
|
+
# Enforce radius limit strictly unless wider_metro=True
|
|
586
|
+
if not wider_metro and dist_m is not None:
|
|
587
|
+
if dist_m > eff_radius:
|
|
588
|
+
logger.debug(
|
|
589
|
+
"Excluding candidate '%s': %.1fm exceeds search radius %dm.",
|
|
590
|
+
biz.name,
|
|
591
|
+
dist_m,
|
|
592
|
+
eff_radius,
|
|
593
|
+
)
|
|
594
|
+
continue
|
|
595
|
+
|
|
596
|
+
filtered.append(biz)
|
|
597
|
+
if len(filtered) >= limit:
|
|
598
|
+
break
|
|
599
|
+
|
|
600
|
+
self._set_cache(key, filtered)
|
|
601
|
+
return filtered
|
|
602
|
+
|
|
603
|
+
async def get_details(self, business_id: str) -> Business | None:
|
|
604
|
+
"""Look up a single OSM element by its leadintel ID (osm_<type>_<id>)."""
|
|
605
|
+
if not business_id.startswith("osm_"):
|
|
606
|
+
return None
|
|
607
|
+
parts = business_id.split("_", 2)
|
|
608
|
+
if len(parts) != 3:
|
|
609
|
+
return None
|
|
610
|
+
_, osm_type, osm_id_str = parts
|
|
611
|
+
try:
|
|
612
|
+
osm_id = int(osm_id_str)
|
|
613
|
+
except ValueError:
|
|
614
|
+
return None
|
|
615
|
+
|
|
616
|
+
# 1. Try Overpass
|
|
617
|
+
query = f"""
|
|
618
|
+
[out:json][timeout:15];
|
|
619
|
+
{osm_type}({osm_id});
|
|
620
|
+
out body center;
|
|
621
|
+
"""
|
|
622
|
+
try:
|
|
623
|
+
data = await self._post_overpass(query)
|
|
624
|
+
elements = data.get("elements", [])
|
|
625
|
+
if elements:
|
|
626
|
+
return _osm_element_to_business(elements[0])
|
|
627
|
+
except Exception as exc: # noqa: BLE001
|
|
628
|
+
logger.warning("Overpass get_details failed for '%s': %s", business_id, exc)
|
|
629
|
+
|
|
630
|
+
# 2. Fall back to Nominatim lookup
|
|
631
|
+
type_prefix = "N" if osm_type == "node" else ("W" if osm_type == "way" else "R")
|
|
632
|
+
lookup_id = f"{type_prefix}{osm_id}"
|
|
633
|
+
try:
|
|
634
|
+
await _nominatim_rate_limit()
|
|
635
|
+
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
|
636
|
+
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
637
|
+
resp = await client.get(
|
|
638
|
+
NOMINATIM_LOOKUP_ENDPOINT,
|
|
639
|
+
params={
|
|
640
|
+
"osm_ids": lookup_id,
|
|
641
|
+
"format": "json",
|
|
642
|
+
"addressdetails": 1,
|
|
643
|
+
"extratags": 1,
|
|
644
|
+
},
|
|
645
|
+
headers=headers,
|
|
646
|
+
)
|
|
647
|
+
resp.raise_for_status()
|
|
648
|
+
items = resp.json()
|
|
649
|
+
if items:
|
|
650
|
+
return _nominatim_element_to_business(items[0])
|
|
651
|
+
except Exception as exc: # noqa: BLE001
|
|
652
|
+
logger.warning("Nominatim lookup failed for '%s': %s", lookup_id, exc)
|
|
653
|
+
|
|
654
|
+
return None
|
|
655
|
+
|
|
656
|
+
# ── Geocoding ─────────────────────────────────────────────────────────────
|
|
657
|
+
|
|
658
|
+
def _parse_coordinates(self, location: str | None) -> tuple[float, float] | None:
|
|
659
|
+
"""Parse 'lat,lon' coordinates string."""
|
|
660
|
+
if not location or "," not in location:
|
|
661
|
+
return None
|
|
662
|
+
parts = [p.strip() for p in location.split(",", 1)]
|
|
663
|
+
if len(parts) == 2:
|
|
664
|
+
try:
|
|
665
|
+
lat = float(parts[0])
|
|
666
|
+
lon = float(parts[1])
|
|
667
|
+
if -90 <= lat <= 90 and -180 <= lon <= 180:
|
|
668
|
+
return lat, lon
|
|
669
|
+
except ValueError:
|
|
670
|
+
pass
|
|
671
|
+
return None
|
|
672
|
+
|
|
673
|
+
async def _resolve_center_and_bbox(
|
|
674
|
+
self,
|
|
675
|
+
location: str | None,
|
|
676
|
+
latitude: float | None = None,
|
|
677
|
+
longitude: float | None = None,
|
|
678
|
+
radius_m: int | None = None,
|
|
679
|
+
) -> tuple[tuple[float, float], tuple[float, float, float, float]] | None:
|
|
680
|
+
"""
|
|
681
|
+
Resolve (center_lat, center_lon) and (south, west, north, east) bbox.
|
|
682
|
+
"""
|
|
683
|
+
r = radius_m if radius_m is not None else self.radius_m
|
|
684
|
+
|
|
685
|
+
# Explicit coordinates
|
|
686
|
+
if latitude is not None and longitude is not None:
|
|
687
|
+
lat = float(latitude)
|
|
688
|
+
lon = float(longitude)
|
|
689
|
+
return (lat, lon), self._bbox_around(lat, lon, r)
|
|
690
|
+
|
|
691
|
+
if not location:
|
|
692
|
+
return None
|
|
693
|
+
|
|
694
|
+
# Location string is "lat,lon"
|
|
695
|
+
coords = self._parse_coordinates(location)
|
|
696
|
+
if coords:
|
|
697
|
+
lat, lon = coords
|
|
698
|
+
return (lat, lon), self._bbox_around(lat, lon, r)
|
|
699
|
+
|
|
700
|
+
# Geocode cache
|
|
701
|
+
loc_key = location.lower().strip()
|
|
702
|
+
cached_entry = _nominatim_geocode_cache.get(loc_key)
|
|
703
|
+
if cached_entry is not None:
|
|
704
|
+
(lat, lon), ts = cached_entry
|
|
705
|
+
if time.monotonic() - ts < NOMINATIM_GEOCODE_TTL:
|
|
706
|
+
logger.debug("Nominatim geocode cache hit for '%s'.", location)
|
|
707
|
+
return (lat, lon), self._bbox_around(lat, lon, r)
|
|
708
|
+
del _nominatim_geocode_cache[loc_key]
|
|
709
|
+
|
|
710
|
+
# Nominatim geocode request
|
|
711
|
+
params = {
|
|
712
|
+
"q": location,
|
|
713
|
+
"format": "json",
|
|
714
|
+
"limit": "1",
|
|
715
|
+
}
|
|
716
|
+
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
|
717
|
+
|
|
718
|
+
try:
|
|
719
|
+
await _nominatim_rate_limit()
|
|
720
|
+
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
721
|
+
resp = await client.get(NOMINATIM_ENDPOINT, params=params, headers=headers)
|
|
722
|
+
resp.raise_for_status()
|
|
723
|
+
results = resp.json()
|
|
724
|
+
|
|
725
|
+
if not results:
|
|
726
|
+
return None
|
|
727
|
+
|
|
728
|
+
top = results[0]
|
|
729
|
+
lat = float(top["lat"])
|
|
730
|
+
lon = float(top["lon"])
|
|
731
|
+
_nominatim_geocode_cache[loc_key] = ((lat, lon), time.monotonic())
|
|
732
|
+
return (lat, lon), self._bbox_around(lat, lon, r)
|
|
733
|
+
|
|
734
|
+
except Exception as exc:
|
|
735
|
+
logger.warning("Nominatim geocoding failed for '%s': %s", location, exc)
|
|
736
|
+
return None
|
|
737
|
+
|
|
738
|
+
async def _resolve_bbox(
|
|
739
|
+
self, location: str | None
|
|
740
|
+
) -> tuple[float, float, float, float] | None:
|
|
741
|
+
"""Return (south, west, north, east) bounding box for a location string."""
|
|
742
|
+
res = await self._resolve_center_and_bbox(location)
|
|
743
|
+
if res is not None:
|
|
744
|
+
return res[1]
|
|
745
|
+
return None
|
|
746
|
+
|
|
747
|
+
def _bbox_around(
|
|
748
|
+
self, lat: float, lon: float, radius_m: int | None = None
|
|
749
|
+
) -> tuple[float, float, float, float]:
|
|
750
|
+
"""Create a square bounding box of radius_m around (lat, lon)."""
|
|
751
|
+
r = radius_m if radius_m is not None else self.radius_m
|
|
752
|
+
delta_lat = r / 111_320
|
|
753
|
+
delta_lon = r / (111_320 * abs(math.cos(math.radians(lat))) or 1)
|
|
754
|
+
return (
|
|
755
|
+
lat - delta_lat,
|
|
756
|
+
lon - delta_lon,
|
|
757
|
+
lat + delta_lat,
|
|
758
|
+
lon + delta_lon,
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
# ── Nominatim fallback search ─────────────────────────────────────────────
|
|
762
|
+
|
|
763
|
+
async def _search_nominatim(
|
|
764
|
+
self,
|
|
765
|
+
query: str,
|
|
766
|
+
location: str,
|
|
767
|
+
category: str | None,
|
|
768
|
+
limit: int,
|
|
769
|
+
bbox: tuple[float, float, float, float] | None = None,
|
|
770
|
+
center: tuple[float, float] | None = None,
|
|
771
|
+
) -> list[Business]:
|
|
772
|
+
"""Direct OpenStreetMap search via Nominatim when Overpass is unavailable."""
|
|
773
|
+
await _nominatim_rate_limit()
|
|
774
|
+
|
|
775
|
+
q_clean = (query or "").strip()
|
|
776
|
+
if q_clean in ("*", "", "all", "any", "business", "businesses"):
|
|
777
|
+
q_clean = None
|
|
778
|
+
|
|
779
|
+
cat_term = None
|
|
780
|
+
if category:
|
|
781
|
+
cat_term = _CATEGORY_TO_TERMS.get(category.lower().strip(), category)
|
|
782
|
+
|
|
783
|
+
primary_term = q_clean or cat_term or "restaurants"
|
|
784
|
+
|
|
785
|
+
params: dict[str, Any] = {
|
|
786
|
+
"format": "jsonv2",
|
|
787
|
+
"limit": limit,
|
|
788
|
+
"addressdetails": 1,
|
|
789
|
+
"extratags": 1,
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
if bbox is not None:
|
|
793
|
+
s, w, n, e = bbox
|
|
794
|
+
params["viewbox"] = f"{w},{s},{e},{n}"
|
|
795
|
+
params["bounded"] = 1
|
|
796
|
+
params["q"] = primary_term
|
|
797
|
+
else:
|
|
798
|
+
coords = self._parse_coordinates(location)
|
|
799
|
+
if coords:
|
|
800
|
+
lat, lon = coords
|
|
801
|
+
s, w, n, e = self._bbox_around(lat, lon)
|
|
802
|
+
params["viewbox"] = f"{w},{s},{e},{n}"
|
|
803
|
+
params["bounded"] = 1
|
|
804
|
+
params["q"] = primary_term
|
|
805
|
+
else:
|
|
806
|
+
params["q"] = f"{primary_term} in {location}"
|
|
807
|
+
|
|
808
|
+
headers = {
|
|
809
|
+
"User-Agent": USER_AGENT,
|
|
810
|
+
"Accept": "application/json",
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
async with httpx.AsyncClient(timeout=REQUEST_TIMEOUT) as client:
|
|
814
|
+
resp = await client.get(NOMINATIM_ENDPOINT, params=params, headers=headers)
|
|
815
|
+
resp.raise_for_status()
|
|
816
|
+
data = resp.json()
|
|
817
|
+
|
|
818
|
+
businesses: list[Business] = []
|
|
819
|
+
seen_names: set[str] = set()
|
|
820
|
+
|
|
821
|
+
if isinstance(data, list):
|
|
822
|
+
for item in data:
|
|
823
|
+
biz = _nominatim_element_to_business(item)
|
|
824
|
+
if biz is None:
|
|
825
|
+
continue
|
|
826
|
+
dedup_key = (biz.name.lower(), (biz.city or "").lower())
|
|
827
|
+
if dedup_key in seen_names:
|
|
828
|
+
continue
|
|
829
|
+
seen_names.add(dedup_key)
|
|
830
|
+
businesses.append(biz)
|
|
831
|
+
if len(businesses) >= limit:
|
|
832
|
+
break
|
|
833
|
+
|
|
834
|
+
return businesses
|
|
835
|
+
|
|
836
|
+
# ── Overpass query builder ────────────────────────────────────────────────
|
|
837
|
+
|
|
838
|
+
def _build_overpass_query(
|
|
839
|
+
self,
|
|
840
|
+
query: str,
|
|
841
|
+
category: str | None,
|
|
842
|
+
bbox: tuple[float, float, float, float],
|
|
843
|
+
limit: int,
|
|
844
|
+
) -> str:
|
|
845
|
+
"""Generate an Overpass QL query for the given parameters."""
|
|
846
|
+
s, w, n, e = bbox
|
|
847
|
+
bbox_str = f"{s},{w},{n},{e}"
|
|
848
|
+
|
|
849
|
+
# Determine which OSM tags to filter on
|
|
850
|
+
filters = self._build_tag_filters(query=query, category=category)
|
|
851
|
+
|
|
852
|
+
# Build union of node + way for each filter (exclude heavy relations for POIs)
|
|
853
|
+
union_parts: list[str] = []
|
|
854
|
+
for f in filters:
|
|
855
|
+
for element_type in ("node", "way"):
|
|
856
|
+
union_parts.append(f'{element_type}{f}({bbox_str});')
|
|
857
|
+
|
|
858
|
+
union_block = "\n ".join(union_parts)
|
|
859
|
+
|
|
860
|
+
return f"""
|
|
861
|
+
[out:json][timeout:{int(REQUEST_TIMEOUT)}];
|
|
862
|
+
(
|
|
863
|
+
{union_block}
|
|
864
|
+
);
|
|
865
|
+
out body center {limit * 3};
|
|
866
|
+
""".strip()
|
|
867
|
+
|
|
868
|
+
@staticmethod
|
|
869
|
+
def _build_tag_filters(query: str, category: str | None) -> list[str]:
|
|
870
|
+
"""
|
|
871
|
+
Map the free-text query + category to Overpass tag filter strings.
|
|
872
|
+
|
|
873
|
+
Returns a list of filter strings like ['["amenity"="restaurant"]',
|
|
874
|
+
'["shop"="bakery"]'] that are interpolated directly into QL.
|
|
875
|
+
"""
|
|
876
|
+
q = (query or "").lower().strip()
|
|
877
|
+
cat = (category or "").lower().strip()
|
|
878
|
+
combined = f"{q} {cat}".strip()
|
|
879
|
+
|
|
880
|
+
# Direct keyword → OSM tag mappings
|
|
881
|
+
_KEYWORD_MAP: list[tuple[str, str]] = [
|
|
882
|
+
("restaurant", '["amenity"="restaurant"]'),
|
|
883
|
+
("cafe", '["amenity"="cafe"]'),
|
|
884
|
+
("coffee", '["amenity"="cafe"]'),
|
|
885
|
+
("bar", '["amenity"="bar"]'),
|
|
886
|
+
("pub", '["amenity"="pub"]'),
|
|
887
|
+
("fast food", '["amenity"="fast_food"]'),
|
|
888
|
+
("hotel", '["tourism"="hotel"]'),
|
|
889
|
+
("hostel", '["tourism"="hostel"]'),
|
|
890
|
+
("bank", '["amenity"="bank"]'),
|
|
891
|
+
("atm", '["amenity"="atm"]'),
|
|
892
|
+
("hospital", '["amenity"="hospital"]'),
|
|
893
|
+
("clinic", '["amenity"="clinic"]'),
|
|
894
|
+
("pharmacy", '["amenity"="pharmacy"]'),
|
|
895
|
+
("dentist", '["amenity"="dentist"]'),
|
|
896
|
+
("school", '["amenity"="school"]'),
|
|
897
|
+
("university", '["amenity"="university"]'),
|
|
898
|
+
("gym", '["leisure"="fitness_centre"]'),
|
|
899
|
+
("fitness", '["leisure"="fitness_centre"]'),
|
|
900
|
+
("supermarket", '["shop"="supermarket"]'),
|
|
901
|
+
("grocery", '["shop"="supermarket"]'),
|
|
902
|
+
("clothes", '["shop"="clothes"]'),
|
|
903
|
+
("fashion", '["shop"="clothes"]'),
|
|
904
|
+
("electronics", '["shop"="electronics"]'),
|
|
905
|
+
("technology", '["office"="it"]'),
|
|
906
|
+
("software", '["office"="software"]'),
|
|
907
|
+
("consulting", '["office"="consulting"]'),
|
|
908
|
+
("law", '["office"="lawyer"]'),
|
|
909
|
+
("lawyer", '["office"="lawyer"]'),
|
|
910
|
+
("accountant", '["office"="accountant"]'),
|
|
911
|
+
("real estate", '["office"="real_estate"]'),
|
|
912
|
+
("estate agent", '["office"="estate_agent"]'),
|
|
913
|
+
("marketing", '["office"="marketing"]'),
|
|
914
|
+
("advertising", '["office"="advertising"]'),
|
|
915
|
+
("coworking", '["amenity"="coworking_space"]'),
|
|
916
|
+
("fuel", '["amenity"="fuel"]'),
|
|
917
|
+
("petrol", '["amenity"="fuel"]'),
|
|
918
|
+
("gas station", '["amenity"="fuel"]'),
|
|
919
|
+
("cinema", '["amenity"="cinema"]'),
|
|
920
|
+
("theatre", '["amenity"="theatre"]'),
|
|
921
|
+
("travel", '["office"="travel_agent"]'),
|
|
922
|
+
("hair", '["shop"="hairdresser"]'),
|
|
923
|
+
("salon", '["shop"="beauty"]'),
|
|
924
|
+
("beauty", '["shop"="beauty"]'),
|
|
925
|
+
("ngo", '["office"="ngo"]'),
|
|
926
|
+
("nonprofit", '["office"="ngo"]'),
|
|
927
|
+
("food", '["amenity"~"restaurant|cafe|fast_food|bar"]'),
|
|
928
|
+
("shop", '["shop"]'),
|
|
929
|
+
("office", '["office"]'),
|
|
930
|
+
("business", '["name"]'),
|
|
931
|
+
]
|
|
932
|
+
|
|
933
|
+
# Try to find a specific match first
|
|
934
|
+
for keyword, tag_filter in _KEYWORD_MAP:
|
|
935
|
+
if keyword in combined:
|
|
936
|
+
return [tag_filter]
|
|
937
|
+
|
|
938
|
+
# Wildcard / generic — return anything with a name tag
|
|
939
|
+
if q in ("*", "", "all", "any"):
|
|
940
|
+
return ['["name"]']
|
|
941
|
+
|
|
942
|
+
# Fall back: name substring search (slow but correct for arbitrary queries)
|
|
943
|
+
safe_q = query.replace('"', "").replace("\\", "")
|
|
944
|
+
return [f'["name"~"{safe_q}",i]']
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
# ── HTTP helpers ──────────────────────────────────────────────────────────
|
|
948
|
+
|
|
949
|
+
async def _run_overpass(
|
|
950
|
+
self,
|
|
951
|
+
query: str,
|
|
952
|
+
category: str | None,
|
|
953
|
+
bbox: tuple[float, float, float, float],
|
|
954
|
+
limit: int,
|
|
955
|
+
) -> list[Business]:
|
|
956
|
+
overpass_query = self._build_overpass_query(
|
|
957
|
+
query=query, category=category, bbox=bbox, limit=limit
|
|
958
|
+
)
|
|
959
|
+
data = await self._post_overpass(overpass_query)
|
|
960
|
+
elements = data.get("elements", [])
|
|
961
|
+
|
|
962
|
+
businesses: list[Business] = []
|
|
963
|
+
seen_names: set[str] = set()
|
|
964
|
+
|
|
965
|
+
for element in elements:
|
|
966
|
+
biz = _osm_element_to_business(element)
|
|
967
|
+
if biz is None:
|
|
968
|
+
continue
|
|
969
|
+
# Deduplicate by name+city within a single query
|
|
970
|
+
dedup_key = (biz.name.lower(), (biz.city or "").lower())
|
|
971
|
+
if dedup_key in seen_names:
|
|
972
|
+
continue
|
|
973
|
+
seen_names.add(dedup_key)
|
|
974
|
+
businesses.append(biz)
|
|
975
|
+
if len(businesses) >= limit:
|
|
976
|
+
break
|
|
977
|
+
|
|
978
|
+
return businesses
|
|
979
|
+
|
|
980
|
+
async def _post_overpass(self, query: str) -> dict:
|
|
981
|
+
"""POST a query to the Overpass API and return parsed JSON."""
|
|
982
|
+
await self._rate_limit()
|
|
983
|
+
headers = {
|
|
984
|
+
"User-Agent": USER_AGENT,
|
|
985
|
+
"Accept": "application/json, */*",
|
|
986
|
+
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
timeout = min(self.request_timeout if hasattr(self, "request_timeout") else REQUEST_TIMEOUT, 12.0)
|
|
990
|
+
# Try primary endpoint plus at most 1 mirror to keep response snappy before Nominatim fallback
|
|
991
|
+
endpoints = [self.endpoint]
|
|
992
|
+
for ep in self.fallback_endpoints:
|
|
993
|
+
if ep != self.endpoint and ep not in endpoints:
|
|
994
|
+
endpoints.append(ep)
|
|
995
|
+
if len(endpoints) >= 2:
|
|
996
|
+
break
|
|
997
|
+
last_exc: Exception | None = None
|
|
998
|
+
|
|
999
|
+
for ep in endpoints:
|
|
1000
|
+
try:
|
|
1001
|
+
async with httpx.AsyncClient(timeout=timeout) as client:
|
|
1002
|
+
resp = await client.post(
|
|
1003
|
+
ep,
|
|
1004
|
+
data={"data": query},
|
|
1005
|
+
headers=headers,
|
|
1006
|
+
)
|
|
1007
|
+
resp.raise_for_status()
|
|
1008
|
+
self._last_request = time.monotonic()
|
|
1009
|
+
return resp.json()
|
|
1010
|
+
except Exception as exc: # noqa: BLE001
|
|
1011
|
+
last_exc = exc
|
|
1012
|
+
logger.warning("Overpass endpoint '%s' failed: %s", ep, exc)
|
|
1013
|
+
|
|
1014
|
+
if last_exc:
|
|
1015
|
+
raise last_exc
|
|
1016
|
+
raise RuntimeError("No Overpass endpoints available")
|
|
1017
|
+
|
|
1018
|
+
async def _rate_limit(self) -> None:
|
|
1019
|
+
"""Sleep if needed to respect the minimum Overpass request interval."""
|
|
1020
|
+
elapsed = time.monotonic() - self._last_request
|
|
1021
|
+
if elapsed < MIN_REQUEST_INTERVAL:
|
|
1022
|
+
await asyncio.sleep(MIN_REQUEST_INTERVAL - elapsed)
|
|
1023
|
+
|
|
1024
|
+
# ── Cache ─────────────────────────────────────────────────────────────────
|
|
1025
|
+
|
|
1026
|
+
def _get_cache(self, key: str) -> list[Business] | None:
|
|
1027
|
+
entry = self._cache.get(key)
|
|
1028
|
+
if entry is None:
|
|
1029
|
+
return None
|
|
1030
|
+
ts, data = entry
|
|
1031
|
+
if time.monotonic() - ts > self._cache_ttl:
|
|
1032
|
+
del self._cache[key]
|
|
1033
|
+
return None
|
|
1034
|
+
return data
|
|
1035
|
+
|
|
1036
|
+
def _set_cache(self, key: str, data: list[Business]) -> None:
|
|
1037
|
+
self._cache[key] = (time.monotonic(), data)
|
|
1038
|
+
|
|
1039
|
+
def clear_cache(self) -> None:
|
|
1040
|
+
self._cache.clear()
|