leadintel 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. leadintel/__init__.py +6 -0
  2. leadintel/__main__.py +6 -0
  3. leadintel/config.py +165 -0
  4. leadintel/data/__init__.py +1 -0
  5. leadintel/data/demo.py +315 -0
  6. leadintel/exporters/__init__.py +6 -0
  7. leadintel/exporters/csv_exporter.py +113 -0
  8. leadintel/exporters/excel_exporter.py +123 -0
  9. leadintel/models/__init__.py +6 -0
  10. leadintel/models/business.py +203 -0
  11. leadintel/models/report.py +31 -0
  12. leadintel/providers/__init__.py +51 -0
  13. leadintel/providers/base.py +54 -0
  14. leadintel/providers/demo_provider.py +65 -0
  15. leadintel/providers/overpass_provider.py +1040 -0
  16. leadintel/providers/registry.py +151 -0
  17. leadintel/server.py +61 -0
  18. leadintel/services/__init__.py +48 -0
  19. leadintel/services/ai_business_insights.py +310 -0
  20. leadintel/services/business_comparison.py +220 -0
  21. leadintel/services/business_report.py +294 -0
  22. leadintel/services/competitor_research.py +293 -0
  23. leadintel/services/lead_scoring.py +385 -0
  24. leadintel/services/local_ai.py +186 -0
  25. leadintel/services/robots_checker.py +81 -0
  26. leadintel/services/seo_auditor.py +281 -0
  27. leadintel/services/social_finder.py +119 -0
  28. leadintel/services/url_validator.py +295 -0
  29. leadintel/services/website_analyzer.py +213 -0
  30. leadintel/services/website_opportunity.py +523 -0
  31. leadintel/storage/__init__.py +12 -0
  32. leadintel/storage/db.py +189 -0
  33. leadintel/storage/path_validator.py +118 -0
  34. leadintel/tools/__init__.py +38 -0
  35. leadintel/tools/analyze_website.py +57 -0
  36. leadintel/tools/check_seo.py +64 -0
  37. leadintel/tools/compare_businesses.py +65 -0
  38. leadintel/tools/export_leads.py +112 -0
  39. leadintel/tools/find_business_website.py +66 -0
  40. leadintel/tools/find_competitors.py +75 -0
  41. leadintel/tools/find_social_profiles.py +85 -0
  42. leadintel/tools/generate_business_insights.py +93 -0
  43. leadintel/tools/generate_business_report.py +92 -0
  44. leadintel/tools/generate_outreach.py +153 -0
  45. leadintel/tools/get_business_details.py +96 -0
  46. leadintel/tools/get_saved_leads.py +58 -0
  47. leadintel/tools/score_lead.py +88 -0
  48. leadintel/tools/search_businesses.py +305 -0
  49. leadintel/tools/website_finder.py +147 -0
  50. leadintel-0.3.0.dist-info/METADATA +351 -0
  51. leadintel-0.3.0.dist-info/RECORD +54 -0
  52. leadintel-0.3.0.dist-info/WHEEL +4 -0
  53. leadintel-0.3.0.dist-info/entry_points.txt +3 -0
  54. leadintel-0.3.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,1040 @@
1
+ """
2
+ OverpassProvider — free business discovery via OpenStreetMap Overpass API.
3
+
4
+ Attribution requirement (ODbL):
5
+ Data © OpenStreetMap contributors, ODbL 1.0.
6
+ https://www.openstreetmap.org/copyright
7
+
8
+ Usage policy:
9
+ - Rate-limit: ≤1 req/s; do not hammer the public endpoint.
10
+ - Use the public endpoint: https://overpass-api.de/api/interpreter
11
+ or a self-hosted instance (set OVERPASS_ENDPOINT env var).
12
+ - Never cache results longer than 7 days to honour OSM data freshness.
13
+ - Always display the ODbL attribution string in exports.
14
+
15
+ This provider:
16
+ * Geocodes location strings via Nominatim (also free, ODbL).
17
+ * Queries Overpass QL for named businesses in a bounding box.
18
+ * Maps OSM tags → Business model fields.
19
+ * Implements a simple in-process TTL cache to avoid hammering the API.
20
+ * Falls back to Nominatim POI search when Overpass is at capacity.
21
+ * Enforces a global 1-second Nominatim rate-limit across all instances.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import asyncio
27
+ import hashlib
28
+ import json
29
+ import logging
30
+ import math
31
+ import time
32
+ from typing import Any
33
+
34
+ import httpx
35
+
36
+ from ..models.business import Business
37
+
38
+ logger = logging.getLogger(__name__)
39
+
40
+ # ── Constants ─────────────────────────────────────────────────────────────────
41
+ OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter"
42
+ NOMINATIM_ENDPOINT = "https://nominatim.openstreetmap.org/search"
43
+ NOMINATIM_REVERSE_ENDPOINT = "https://nominatim.openstreetmap.org/reverse"
44
+ NOMINATIM_LOOKUP_ENDPOINT = "https://nominatim.openstreetmap.org/lookup"
45
+ OSM_ATTRIBUTION = "Data © OpenStreetMap contributors, ODbL 1.0"
46
+ # Descriptive User-Agent per Nominatim usage policy: https://operations.osmfoundation.org/policies/nominatim/
47
+ USER_AGENT = "LeadIntel/0.2.0 (open-source B2B lead-generation MCP server; https://github.com/example/leadintel)"
48
+
49
+ OVERPASS_FALLBACK_ENDPOINTS = [
50
+ "https://lz4.overpass-api.de/api/interpreter",
51
+ "https://z.overpass-api.de/api/interpreter",
52
+ "https://overpass.private.coffee/api/interpreter",
53
+ ]
54
+
55
+ # Default search radius around a geocoded point, in metres (6km covers central urban core)
56
+ DEFAULT_RADIUS_M = 6_000
57
+ # Max radius to keep queries fast
58
+ MAX_RADIUS_M = 50_000
59
+ # Overpass request timeout (seconds)
60
+ REQUEST_TIMEOUT = 30.0
61
+ # Minimum seconds between consecutive Overpass requests (rate-limit)
62
+ MIN_REQUEST_INTERVAL = 1.1
63
+ # Nominatim usage policy: https://operations.osmfoundation.org/policies/nominatim/
64
+ # Minimum 1-second interval between any Nominatim requests (global across all instances)
65
+ NOMINATIM_MIN_INTERVAL = 1.0
66
+ # Simple in-process cache TTL (seconds); never exceed 7 days (604 800 s)
67
+ CACHE_TTL = 3_600 # 1 hour
68
+ # Nominatim geocode cache TTL — city name → coords; reuse to avoid repeat requests
69
+ NOMINATIM_GEOCODE_TTL = 86_400 # 24 hours; city locations do not change
70
+
71
+ # Global Nominatim rate-limit clock — shared across all OverpassProvider instances
72
+ # so concurrent searches in the same process cannot exceed the usage policy limit.
73
+ _nominatim_last_request: float = 0.0
74
+ _nominatim_lock: asyncio.Lock | None = None # initialised lazily; must be in an event loop
75
+
76
+ # Global Nominatim geocode cache: location string → (coords (lat, lon), timestamp)
77
+ _nominatim_geocode_cache: dict[str, tuple[tuple[float, float], float]] = {}
78
+
79
+
80
+ def haversine_distance_meters(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
81
+ """Calculate the great-circle distance between two points on Earth in metres."""
82
+ r = 6_371_000.0 # Earth radius in metres
83
+ phi1 = math.radians(lat1)
84
+ phi2 = math.radians(lat2)
85
+ delta_phi = math.radians(lat2 - lat1)
86
+ delta_lambda = math.radians(lon2 - lon1)
87
+
88
+ a = (
89
+ math.sin(delta_phi / 2.0) ** 2
90
+ + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2.0) ** 2
91
+ )
92
+ c = 2.0 * math.atan2(math.sqrt(a), math.sqrt(1.0 - a))
93
+ return r * c
94
+
95
+
96
+ async def _nominatim_rate_limit() -> None:
97
+ """
98
+ Enforce a global minimum 1-second gap between all Nominatim HTTP requests.
99
+
100
+ This is shared across every OverpassProvider instance in the same process so
101
+ concurrent tool calls never violate the Nominatim Acceptable Use Policy
102
+ (https://operations.osmfoundation.org/policies/nominatim/).
103
+
104
+ Uses a module-level asyncio.Lock (initialised lazily on first call so it is
105
+ always created inside a running event loop) to serialise concurrent callers.
106
+ """
107
+ global _nominatim_last_request, _nominatim_lock # noqa: PLW0603
108
+ if _nominatim_lock is None:
109
+ _nominatim_lock = asyncio.Lock()
110
+ async with _nominatim_lock:
111
+ elapsed = time.monotonic() - _nominatim_last_request
112
+ if elapsed < NOMINATIM_MIN_INTERVAL:
113
+ await asyncio.sleep(NOMINATIM_MIN_INTERVAL - elapsed)
114
+ _nominatim_last_request = time.monotonic()
115
+
116
+
117
+ # ── OSM tag → category mapping ────────────────────────────────────────────────
118
+ _OSM_AMENITY_TO_CATEGORY: dict[str, str] = {
119
+ "restaurant": "Food & Beverage",
120
+ "cafe": "Food & Beverage",
121
+ "bar": "Food & Beverage",
122
+ "pub": "Food & Beverage",
123
+ "fast_food": "Food & Beverage",
124
+ "hospital": "Healthcare",
125
+ "clinic": "Healthcare",
126
+ "pharmacy": "Healthcare",
127
+ "dentist": "Healthcare",
128
+ "school": "Education",
129
+ "university": "Education",
130
+ "college": "Education",
131
+ "bank": "Financial Services",
132
+ "atm": "Financial Services",
133
+ "hotel": "Hospitality",
134
+ "hostel": "Hospitality",
135
+ "guest_house": "Hospitality",
136
+ "gym": "Health & Fitness",
137
+ "fuel": "Automotive",
138
+ "car_wash": "Automotive",
139
+ "cinema": "Entertainment",
140
+ "theatre": "Entertainment",
141
+ "library": "Education",
142
+ "post_office": "Government Services",
143
+ "police": "Government Services",
144
+ "fire_station": "Government Services",
145
+ "place_of_worship": "Religion",
146
+ "marketplace": "Retail",
147
+ "supermarket": "Retail",
148
+ "convenience": "Retail",
149
+ "hairdresser": "Personal Care",
150
+ "beauty": "Personal Care",
151
+ "laundry": "Personal Care",
152
+ "shop": "Retail",
153
+ "office": "Professional Services",
154
+ "coworking_space": "Professional Services",
155
+ "it": "Technology",
156
+ "accountant": "Professional Services",
157
+ "lawyer": "Legal Services",
158
+ "estate_agent": "Real Estate",
159
+ "travel_agency": "Travel",
160
+ "car_rental": "Automotive",
161
+ "charging_station": "Automotive",
162
+ }
163
+
164
+ _OSM_SHOP_TO_CATEGORY: dict[str, str] = {
165
+ "supermarket": "Retail – Grocery",
166
+ "clothes": "Retail – Fashion",
167
+ "electronics": "Retail – Electronics",
168
+ "furniture": "Retail – Furniture",
169
+ "hardware": "Retail – Hardware",
170
+ "bakery": "Food & Beverage",
171
+ "butcher": "Food & Beverage",
172
+ "florist": "Retail – Florist",
173
+ "books": "Retail – Books",
174
+ "sports": "Retail – Sports",
175
+ "jewelry": "Retail – Jewellery",
176
+ "computer": "Technology",
177
+ "mobile_phone": "Technology",
178
+ "car": "Automotive – Dealer",
179
+ "bicycle": "Retail – Bicycle",
180
+ "optician": "Healthcare – Optician",
181
+ "chemist": "Healthcare – Pharmacy",
182
+ }
183
+
184
+ _OSM_OFFICE_TO_CATEGORY: dict[str, str] = {
185
+ "it": "Technology",
186
+ "software": "Technology",
187
+ "company": "Professional Services",
188
+ "consulting": "Consulting",
189
+ "lawyer": "Legal Services",
190
+ "accountant": "Accounting",
191
+ "architect": "Architecture",
192
+ "engineer": "Engineering",
193
+ "financial": "Financial Services",
194
+ "insurance": "Insurance",
195
+ "marketing": "Marketing",
196
+ "advertising": "Marketing",
197
+ "real_estate": "Real Estate",
198
+ "travel_agent": "Travel",
199
+ "government": "Government",
200
+ "ngo": "Non-Profit",
201
+ }
202
+
203
+
204
+ def _osm_tags_to_category(tags: dict[str, str]) -> str | None:
205
+ amenity = tags.get("amenity", "")
206
+ shop = tags.get("shop", "")
207
+ office = tags.get("office", "")
208
+
209
+ if amenity:
210
+ return _OSM_AMENITY_TO_CATEGORY.get(amenity, amenity.replace("_", " ").title())
211
+ if shop:
212
+ return _OSM_SHOP_TO_CATEGORY.get(shop, shop.replace("_", " ").title())
213
+ if office:
214
+ return _OSM_OFFICE_TO_CATEGORY.get(office, office.replace("_", " ").title())
215
+
216
+ craft = tags.get("craft", "")
217
+ if craft:
218
+ return craft.replace("_", " ").title()
219
+
220
+ tourism = tags.get("tourism", "")
221
+ if tourism:
222
+ return tourism.replace("_", " ").title()
223
+
224
+ healthcare = tags.get("healthcare", "")
225
+ if healthcare:
226
+ return "Healthcare"
227
+
228
+ leisure = tags.get("leisure", "")
229
+ if leisure:
230
+ return leisure.replace("_", " ").title()
231
+
232
+ return None
233
+
234
+
235
+ def is_valid_business_element(element: dict[str, Any]) -> bool:
236
+ """
237
+ Validate whether an OSM element is an actual business or commercial POI.
238
+ Strictly filters out roads, ways, railways, natural boundaries, bus stops, and infrastructure.
239
+ """
240
+ tags: dict[str, str] = element.get("tags", {})
241
+ name = tags.get("name") or tags.get("brand") or tags.get("operator")
242
+ if not name:
243
+ return False
244
+
245
+ # Commercial classification tags
246
+ has_commercial_tag = any(
247
+ k in tags
248
+ for k in (
249
+ "amenity", "shop", "office", "craft", "tourism",
250
+ "leisure", "healthcare", "commercial", "brand", "operator"
251
+ )
252
+ )
253
+
254
+ # Reject road ways and transport infrastructure
255
+ highway = tags.get("highway")
256
+ if highway:
257
+ # A road or street is NOT a business unless it carries a specific business amenity/shop/office
258
+ if not any(k in tags for k in ("shop", "office", "craft")):
259
+ amenity = tags.get("amenity")
260
+ if amenity not in ("restaurant", "cafe", "fast_food", "fuel", "bar", "pub", "pharmacy", "bank"):
261
+ return False
262
+
263
+ # Reject non-business infrastructure
264
+ if any(k in tags for k in ("railway", "waterway", "natural", "boundary", "barrier", "traffic_calming")):
265
+ if not has_commercial_tag:
266
+ return False
267
+
268
+ # Reject geographic boundaries, suburbs, or cities
269
+ if "place" in tags and not has_commercial_tag:
270
+ return False
271
+
272
+ return True
273
+
274
+
275
+ def _osm_element_to_business(element: dict[str, Any]) -> Business | None:
276
+ """Convert a raw Overpass API element to a Business. Returns None if unusable or non-business."""
277
+ if not is_valid_business_element(element):
278
+ return None
279
+
280
+ tags: dict[str, str] = element.get("tags", {})
281
+
282
+ name = tags.get("name") or tags.get("brand") or tags.get("operator")
283
+ if not name:
284
+ return None
285
+
286
+ # Coordinates — node has lat/lon directly; way/relation use centre
287
+ if element["type"] == "node":
288
+ lat = element.get("lat")
289
+ lon = element.get("lon")
290
+ else:
291
+ centre = element.get("center", {})
292
+ lat = centre.get("lat")
293
+ lon = centre.get("lon")
294
+
295
+ osm_id = f"osm_{element['type']}_{element['id']}"
296
+
297
+ # Address
298
+ addr_parts = [
299
+ tags.get("addr:housenumber", ""),
300
+ tags.get("addr:street", ""),
301
+ ]
302
+ street = " ".join(p for p in addr_parts if p).strip()
303
+ city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village")
304
+ state = tags.get("addr:state") or tags.get("addr:province")
305
+ country = tags.get("addr:country")
306
+ postal_code = tags.get("addr:postcode")
307
+
308
+ address_parts = [p for p in [street, city, state, postal_code, country] if p]
309
+ address = ", ".join(address_parts) if address_parts else None
310
+
311
+ return Business(
312
+ id=osm_id,
313
+ name=name,
314
+ category=_osm_tags_to_category(tags),
315
+ address=address,
316
+ city=city,
317
+ state=state,
318
+ country=country,
319
+ postal_code=postal_code,
320
+ latitude=lat,
321
+ longitude=lon,
322
+ phone=tags.get("phone") or tags.get("contact:phone"),
323
+ email=tags.get("email") or tags.get("contact:email"),
324
+ website=tags.get("website") or tags.get("contact:website") or tags.get("url"),
325
+ source="openstreetmap",
326
+ is_ai_generated=False,
327
+ osm_tags=tags,
328
+ )
329
+
330
+
331
+ _CATEGORY_TO_TERMS: dict[str, str] = {
332
+ "food & beverage": "restaurants",
333
+ "healthcare": "hospitals",
334
+ "education": "schools",
335
+ "financial services": "banks",
336
+ "hospitality": "hotels",
337
+ "health & fitness": "fitness",
338
+ "automotive": "car",
339
+ "entertainment": "cinema",
340
+ "retail": "supermarkets",
341
+ "personal care": "beauty",
342
+ "professional services": "offices",
343
+ "technology": "software",
344
+ "legal services": "lawyer",
345
+ "real estate": "real_estate",
346
+ "travel": "travel_agency",
347
+ }
348
+
349
+
350
+ def _nominatim_element_to_business(element: dict[str, Any]) -> Business | None:
351
+ """Convert a raw Nominatim search/lookup element to a Business. Returns None if unusable."""
352
+ osm_type = element.get("osm_type")
353
+ osm_id = element.get("osm_id")
354
+ if not osm_type or not osm_id:
355
+ return None
356
+
357
+ cat_class = element.get("category") or element.get("class", "")
358
+ if cat_class in ("highway", "boundary", "place", "waterway", "railway", "natural"):
359
+ return None
360
+
361
+ name = element.get("name")
362
+ if not name:
363
+ display_name = element.get("display_name", "")
364
+ name = display_name.split(",")[0].strip() if display_name else None
365
+ if not name:
366
+ return None
367
+
368
+ try:
369
+ lat = float(element["lat"]) if "lat" in element else None
370
+ lon = float(element["lon"]) if "lon" in element else None
371
+ except (ValueError, TypeError):
372
+ lat = None
373
+ lon = None
374
+
375
+ addr: dict[str, Any] = element.get("address") or {}
376
+ street = addr.get("road") or addr.get("street")
377
+ city = (
378
+ addr.get("city")
379
+ or addr.get("town")
380
+ or addr.get("village")
381
+ or addr.get("suburb")
382
+ or addr.get("county")
383
+ )
384
+ state = addr.get("state") or addr.get("province")
385
+ country = addr.get("country")
386
+ postal_code = addr.get("postcode")
387
+
388
+ address_parts = [p for p in [street, city, state, postal_code, country] if p]
389
+ formatted_address = ", ".join(address_parts) if address_parts else element.get("display_name")
390
+
391
+ extratags: dict[str, Any] = element.get("extratags") or {}
392
+ phone = (
393
+ extratags.get("phone")
394
+ or extratags.get("contact:phone")
395
+ or element.get("phone")
396
+ )
397
+ email = (
398
+ extratags.get("email")
399
+ or extratags.get("contact:email")
400
+ or element.get("email")
401
+ )
402
+ website = (
403
+ extratags.get("website")
404
+ or extratags.get("contact:website")
405
+ or extratags.get("url")
406
+ or element.get("website")
407
+ )
408
+
409
+ cat_type = element.get("type", "")
410
+ category = None
411
+ if cat_class:
412
+ category = _osm_tags_to_category({cat_class: cat_type})
413
+ if not category and cat_type:
414
+ category = _osm_tags_to_category({"amenity": cat_type, "shop": cat_type, "office": cat_type})
415
+ if not category and cat_type:
416
+ category = cat_type.replace("_", " ").title()
417
+
418
+ leadintel_id = f"osm_{osm_type}_{osm_id}"
419
+
420
+ return Business(
421
+ id=leadintel_id,
422
+ name=name,
423
+ category=category,
424
+ address=formatted_address,
425
+ city=city,
426
+ state=state,
427
+ country=country,
428
+ postal_code=postal_code,
429
+ latitude=lat,
430
+ longitude=lon,
431
+ phone=phone,
432
+ email=email,
433
+ website=website,
434
+ source="openstreetmap",
435
+ is_ai_generated=False,
436
+ osm_tags={"class": cat_class, "type": cat_type, **extratags},
437
+ )
438
+
439
+
440
+ def _cache_key(
441
+ query: str,
442
+ location: str | None,
443
+ category: str | None,
444
+ limit: int,
445
+ lat: float | None = None,
446
+ lon: float | None = None,
447
+ radius: int | None = None,
448
+ ) -> str:
449
+ raw = json.dumps(
450
+ {
451
+ "q": query,
452
+ "loc": location,
453
+ "cat": category,
454
+ "lim": limit,
455
+ "lat": lat,
456
+ "lon": lon,
457
+ "rad": radius,
458
+ },
459
+ sort_keys=True,
460
+ )
461
+ return hashlib.md5(raw.encode()).hexdigest() # noqa: S324 — non-crypto use
462
+
463
+
464
+ class OverpassProvider:
465
+ """
466
+ Business discovery via the free Overpass API (OpenStreetMap data).
467
+
468
+ Attributes:
469
+ name: Provider identifier shown in responses.
470
+ requires_auth: False — no API key needed.
471
+ endpoint: Overpass interpreter URL (default: public instance).
472
+ radius_m: Search radius in metres around a geocoded location.
473
+ """
474
+
475
+ name: str = "openstreetmap"
476
+ requires_auth: bool = False
477
+
478
+ def __init__(
479
+ self,
480
+ endpoint: str = OVERPASS_ENDPOINT,
481
+ radius_m: int = DEFAULT_RADIUS_M,
482
+ cache_ttl: int = CACHE_TTL,
483
+ fallback_endpoints: list[str] | None = None,
484
+ ) -> None:
485
+ self.endpoint = endpoint
486
+ self.radius_m = min(radius_m, MAX_RADIUS_M)
487
+ self._cache: dict[str, tuple[float, list[Business]]] = {}
488
+ self._cache_ttl = cache_ttl
489
+ self._last_request: float = 0.0
490
+ self.fallback_endpoints = (
491
+ fallback_endpoints
492
+ if fallback_endpoints is not None
493
+ else list(OVERPASS_FALLBACK_ENDPOINTS)
494
+ )
495
+ self.last_search_center: dict[str, float] | None = None
496
+ self.last_radius_meters: int | None = None
497
+
498
+ # ── Public interface ──────────────────────────────────────────────────────
499
+
500
+ async def search(
501
+ self,
502
+ query: str,
503
+ location: str | None = None,
504
+ category: str | None = None,
505
+ limit: int = 10,
506
+ latitude: float | None = None,
507
+ longitude: float | None = None,
508
+ radius_meters: int | None = None,
509
+ wider_metro: bool = False,
510
+ **kwargs: Any,
511
+ ) -> list[Business]:
512
+ """Search OSM for businesses matching the query near the location."""
513
+ if not location and (latitude is None or longitude is None):
514
+ logger.warning("No location or coordinates provided; skipping OSM search.")
515
+ return []
516
+
517
+ eff_radius = (
518
+ (radius_meters if radius_meters is not None else 25_000)
519
+ if wider_metro
520
+ else (radius_meters if radius_meters is not None else self.radius_m)
521
+ )
522
+ eff_radius = min(eff_radius, MAX_RADIUS_M)
523
+
524
+ center_and_bbox = await self._resolve_center_and_bbox(
525
+ location=location,
526
+ latitude=latitude,
527
+ longitude=longitude,
528
+ radius_m=eff_radius,
529
+ )
530
+ if center_and_bbox is None:
531
+ logger.warning("Could not geocode location '%s'; skipping OSM search.", location)
532
+ return []
533
+
534
+ center, bbox = center_and_bbox
535
+ center_lat, center_lon = center
536
+ self.last_search_center = {"latitude": center_lat, "longitude": center_lon}
537
+ self.last_radius_meters = eff_radius
538
+
539
+ key = _cache_key(query, location, category, limit, center_lat, center_lon, eff_radius)
540
+ cached = self._get_cache(key)
541
+ if cached is not None:
542
+ logger.debug("Cache hit for key %s", key)
543
+ return cached
544
+
545
+ raw_businesses: list[Business] = []
546
+ overpass_err: Exception | None = None
547
+
548
+ # 1. Attempt Overpass query
549
+ try:
550
+ raw_businesses = await self._run_overpass(
551
+ query=query, category=category, bbox=bbox, limit=limit * 2
552
+ )
553
+ except Exception as exc: # noqa: BLE001
554
+ overpass_err = exc
555
+ logger.warning("Overpass search failed (%s); trying Nominatim fallback.", exc)
556
+
557
+ # 2. If Overpass failed or returned no results, fall back to Nominatim live search
558
+ if not raw_businesses:
559
+ try:
560
+ raw_businesses = await self._search_nominatim(
561
+ query=query,
562
+ location=location or f"{center_lat},{center_lon}",
563
+ category=category,
564
+ limit=limit * 2,
565
+ bbox=bbox,
566
+ center=center,
567
+ )
568
+ except Exception as exc: # noqa: BLE001
569
+ logger.warning("Nominatim live business search failed: %s", exc)
570
+ if overpass_err:
571
+ raise RuntimeError(
572
+ f"OpenStreetMap search failed (Overpass: {overpass_err}; Nominatim: {exc})"
573
+ ) from overpass_err
574
+
575
+ # 3. Calculate distance and strictly enforce radius
576
+ filtered: list[Business] = []
577
+ for biz in raw_businesses:
578
+ if biz.latitude is not None and biz.longitude is not None:
579
+ dist_m = haversine_distance_meters(center_lat, center_lon, biz.latitude, biz.longitude)
580
+ biz.distance_meters = round(dist_m, 1)
581
+ biz.distance_km = round(dist_m / 1000.0, 2)
582
+ else:
583
+ dist_m = None
584
+
585
+ # Enforce radius limit strictly unless wider_metro=True
586
+ if not wider_metro and dist_m is not None:
587
+ if dist_m > eff_radius:
588
+ logger.debug(
589
+ "Excluding candidate '%s': %.1fm exceeds search radius %dm.",
590
+ biz.name,
591
+ dist_m,
592
+ eff_radius,
593
+ )
594
+ continue
595
+
596
+ filtered.append(biz)
597
+ if len(filtered) >= limit:
598
+ break
599
+
600
+ self._set_cache(key, filtered)
601
+ return filtered
602
+
603
+ async def get_details(self, business_id: str) -> Business | None:
604
+ """Look up a single OSM element by its leadintel ID (osm_<type>_<id>)."""
605
+ if not business_id.startswith("osm_"):
606
+ return None
607
+ parts = business_id.split("_", 2)
608
+ if len(parts) != 3:
609
+ return None
610
+ _, osm_type, osm_id_str = parts
611
+ try:
612
+ osm_id = int(osm_id_str)
613
+ except ValueError:
614
+ return None
615
+
616
+ # 1. Try Overpass
617
+ query = f"""
618
+ [out:json][timeout:15];
619
+ {osm_type}({osm_id});
620
+ out body center;
621
+ """
622
+ try:
623
+ data = await self._post_overpass(query)
624
+ elements = data.get("elements", [])
625
+ if elements:
626
+ return _osm_element_to_business(elements[0])
627
+ except Exception as exc: # noqa: BLE001
628
+ logger.warning("Overpass get_details failed for '%s': %s", business_id, exc)
629
+
630
+ # 2. Fall back to Nominatim lookup
631
+ type_prefix = "N" if osm_type == "node" else ("W" if osm_type == "way" else "R")
632
+ lookup_id = f"{type_prefix}{osm_id}"
633
+ try:
634
+ await _nominatim_rate_limit()
635
+ headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
636
+ async with httpx.AsyncClient(timeout=15.0) as client:
637
+ resp = await client.get(
638
+ NOMINATIM_LOOKUP_ENDPOINT,
639
+ params={
640
+ "osm_ids": lookup_id,
641
+ "format": "json",
642
+ "addressdetails": 1,
643
+ "extratags": 1,
644
+ },
645
+ headers=headers,
646
+ )
647
+ resp.raise_for_status()
648
+ items = resp.json()
649
+ if items:
650
+ return _nominatim_element_to_business(items[0])
651
+ except Exception as exc: # noqa: BLE001
652
+ logger.warning("Nominatim lookup failed for '%s': %s", lookup_id, exc)
653
+
654
+ return None
655
+
656
+ # ── Geocoding ─────────────────────────────────────────────────────────────
657
+
658
+ def _parse_coordinates(self, location: str | None) -> tuple[float, float] | None:
659
+ """Parse 'lat,lon' coordinates string."""
660
+ if not location or "," not in location:
661
+ return None
662
+ parts = [p.strip() for p in location.split(",", 1)]
663
+ if len(parts) == 2:
664
+ try:
665
+ lat = float(parts[0])
666
+ lon = float(parts[1])
667
+ if -90 <= lat <= 90 and -180 <= lon <= 180:
668
+ return lat, lon
669
+ except ValueError:
670
+ pass
671
+ return None
672
+
673
+ async def _resolve_center_and_bbox(
674
+ self,
675
+ location: str | None,
676
+ latitude: float | None = None,
677
+ longitude: float | None = None,
678
+ radius_m: int | None = None,
679
+ ) -> tuple[tuple[float, float], tuple[float, float, float, float]] | None:
680
+ """
681
+ Resolve (center_lat, center_lon) and (south, west, north, east) bbox.
682
+ """
683
+ r = radius_m if radius_m is not None else self.radius_m
684
+
685
+ # Explicit coordinates
686
+ if latitude is not None and longitude is not None:
687
+ lat = float(latitude)
688
+ lon = float(longitude)
689
+ return (lat, lon), self._bbox_around(lat, lon, r)
690
+
691
+ if not location:
692
+ return None
693
+
694
+ # Location string is "lat,lon"
695
+ coords = self._parse_coordinates(location)
696
+ if coords:
697
+ lat, lon = coords
698
+ return (lat, lon), self._bbox_around(lat, lon, r)
699
+
700
+ # Geocode cache
701
+ loc_key = location.lower().strip()
702
+ cached_entry = _nominatim_geocode_cache.get(loc_key)
703
+ if cached_entry is not None:
704
+ (lat, lon), ts = cached_entry
705
+ if time.monotonic() - ts < NOMINATIM_GEOCODE_TTL:
706
+ logger.debug("Nominatim geocode cache hit for '%s'.", location)
707
+ return (lat, lon), self._bbox_around(lat, lon, r)
708
+ del _nominatim_geocode_cache[loc_key]
709
+
710
+ # Nominatim geocode request
711
+ params = {
712
+ "q": location,
713
+ "format": "json",
714
+ "limit": "1",
715
+ }
716
+ headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
717
+
718
+ try:
719
+ await _nominatim_rate_limit()
720
+ async with httpx.AsyncClient(timeout=15.0) as client:
721
+ resp = await client.get(NOMINATIM_ENDPOINT, params=params, headers=headers)
722
+ resp.raise_for_status()
723
+ results = resp.json()
724
+
725
+ if not results:
726
+ return None
727
+
728
+ top = results[0]
729
+ lat = float(top["lat"])
730
+ lon = float(top["lon"])
731
+ _nominatim_geocode_cache[loc_key] = ((lat, lon), time.monotonic())
732
+ return (lat, lon), self._bbox_around(lat, lon, r)
733
+
734
+ except Exception as exc:
735
+ logger.warning("Nominatim geocoding failed for '%s': %s", location, exc)
736
+ return None
737
+
738
+ async def _resolve_bbox(
739
+ self, location: str | None
740
+ ) -> tuple[float, float, float, float] | None:
741
+ """Return (south, west, north, east) bounding box for a location string."""
742
+ res = await self._resolve_center_and_bbox(location)
743
+ if res is not None:
744
+ return res[1]
745
+ return None
746
+
747
+ def _bbox_around(
748
+ self, lat: float, lon: float, radius_m: int | None = None
749
+ ) -> tuple[float, float, float, float]:
750
+ """Create a square bounding box of radius_m around (lat, lon)."""
751
+ r = radius_m if radius_m is not None else self.radius_m
752
+ delta_lat = r / 111_320
753
+ delta_lon = r / (111_320 * abs(math.cos(math.radians(lat))) or 1)
754
+ return (
755
+ lat - delta_lat,
756
+ lon - delta_lon,
757
+ lat + delta_lat,
758
+ lon + delta_lon,
759
+ )
760
+
761
+ # ── Nominatim fallback search ─────────────────────────────────────────────
762
+
763
+ async def _search_nominatim(
764
+ self,
765
+ query: str,
766
+ location: str,
767
+ category: str | None,
768
+ limit: int,
769
+ bbox: tuple[float, float, float, float] | None = None,
770
+ center: tuple[float, float] | None = None,
771
+ ) -> list[Business]:
772
+ """Direct OpenStreetMap search via Nominatim when Overpass is unavailable."""
773
+ await _nominatim_rate_limit()
774
+
775
+ q_clean = (query or "").strip()
776
+ if q_clean in ("*", "", "all", "any", "business", "businesses"):
777
+ q_clean = None
778
+
779
+ cat_term = None
780
+ if category:
781
+ cat_term = _CATEGORY_TO_TERMS.get(category.lower().strip(), category)
782
+
783
+ primary_term = q_clean or cat_term or "restaurants"
784
+
785
+ params: dict[str, Any] = {
786
+ "format": "jsonv2",
787
+ "limit": limit,
788
+ "addressdetails": 1,
789
+ "extratags": 1,
790
+ }
791
+
792
+ if bbox is not None:
793
+ s, w, n, e = bbox
794
+ params["viewbox"] = f"{w},{s},{e},{n}"
795
+ params["bounded"] = 1
796
+ params["q"] = primary_term
797
+ else:
798
+ coords = self._parse_coordinates(location)
799
+ if coords:
800
+ lat, lon = coords
801
+ s, w, n, e = self._bbox_around(lat, lon)
802
+ params["viewbox"] = f"{w},{s},{e},{n}"
803
+ params["bounded"] = 1
804
+ params["q"] = primary_term
805
+ else:
806
+ params["q"] = f"{primary_term} in {location}"
807
+
808
+ headers = {
809
+ "User-Agent": USER_AGENT,
810
+ "Accept": "application/json",
811
+ }
812
+
813
+ async with httpx.AsyncClient(timeout=REQUEST_TIMEOUT) as client:
814
+ resp = await client.get(NOMINATIM_ENDPOINT, params=params, headers=headers)
815
+ resp.raise_for_status()
816
+ data = resp.json()
817
+
818
+ businesses: list[Business] = []
819
+ seen_names: set[str] = set()
820
+
821
+ if isinstance(data, list):
822
+ for item in data:
823
+ biz = _nominatim_element_to_business(item)
824
+ if biz is None:
825
+ continue
826
+ dedup_key = (biz.name.lower(), (biz.city or "").lower())
827
+ if dedup_key in seen_names:
828
+ continue
829
+ seen_names.add(dedup_key)
830
+ businesses.append(biz)
831
+ if len(businesses) >= limit:
832
+ break
833
+
834
+ return businesses
835
+
836
+ # ── Overpass query builder ────────────────────────────────────────────────
837
+
838
+ def _build_overpass_query(
839
+ self,
840
+ query: str,
841
+ category: str | None,
842
+ bbox: tuple[float, float, float, float],
843
+ limit: int,
844
+ ) -> str:
845
+ """Generate an Overpass QL query for the given parameters."""
846
+ s, w, n, e = bbox
847
+ bbox_str = f"{s},{w},{n},{e}"
848
+
849
+ # Determine which OSM tags to filter on
850
+ filters = self._build_tag_filters(query=query, category=category)
851
+
852
+ # Build union of node + way for each filter (exclude heavy relations for POIs)
853
+ union_parts: list[str] = []
854
+ for f in filters:
855
+ for element_type in ("node", "way"):
856
+ union_parts.append(f'{element_type}{f}({bbox_str});')
857
+
858
+ union_block = "\n ".join(union_parts)
859
+
860
+ return f"""
861
+ [out:json][timeout:{int(REQUEST_TIMEOUT)}];
862
+ (
863
+ {union_block}
864
+ );
865
+ out body center {limit * 3};
866
+ """.strip()
867
+
868
+ @staticmethod
869
+ def _build_tag_filters(query: str, category: str | None) -> list[str]:
870
+ """
871
+ Map the free-text query + category to Overpass tag filter strings.
872
+
873
+ Returns a list of filter strings like ['["amenity"="restaurant"]',
874
+ '["shop"="bakery"]'] that are interpolated directly into QL.
875
+ """
876
+ q = (query or "").lower().strip()
877
+ cat = (category or "").lower().strip()
878
+ combined = f"{q} {cat}".strip()
879
+
880
+ # Direct keyword → OSM tag mappings
881
+ _KEYWORD_MAP: list[tuple[str, str]] = [
882
+ ("restaurant", '["amenity"="restaurant"]'),
883
+ ("cafe", '["amenity"="cafe"]'),
884
+ ("coffee", '["amenity"="cafe"]'),
885
+ ("bar", '["amenity"="bar"]'),
886
+ ("pub", '["amenity"="pub"]'),
887
+ ("fast food", '["amenity"="fast_food"]'),
888
+ ("hotel", '["tourism"="hotel"]'),
889
+ ("hostel", '["tourism"="hostel"]'),
890
+ ("bank", '["amenity"="bank"]'),
891
+ ("atm", '["amenity"="atm"]'),
892
+ ("hospital", '["amenity"="hospital"]'),
893
+ ("clinic", '["amenity"="clinic"]'),
894
+ ("pharmacy", '["amenity"="pharmacy"]'),
895
+ ("dentist", '["amenity"="dentist"]'),
896
+ ("school", '["amenity"="school"]'),
897
+ ("university", '["amenity"="university"]'),
898
+ ("gym", '["leisure"="fitness_centre"]'),
899
+ ("fitness", '["leisure"="fitness_centre"]'),
900
+ ("supermarket", '["shop"="supermarket"]'),
901
+ ("grocery", '["shop"="supermarket"]'),
902
+ ("clothes", '["shop"="clothes"]'),
903
+ ("fashion", '["shop"="clothes"]'),
904
+ ("electronics", '["shop"="electronics"]'),
905
+ ("technology", '["office"="it"]'),
906
+ ("software", '["office"="software"]'),
907
+ ("consulting", '["office"="consulting"]'),
908
+ ("law", '["office"="lawyer"]'),
909
+ ("lawyer", '["office"="lawyer"]'),
910
+ ("accountant", '["office"="accountant"]'),
911
+ ("real estate", '["office"="real_estate"]'),
912
+ ("estate agent", '["office"="estate_agent"]'),
913
+ ("marketing", '["office"="marketing"]'),
914
+ ("advertising", '["office"="advertising"]'),
915
+ ("coworking", '["amenity"="coworking_space"]'),
916
+ ("fuel", '["amenity"="fuel"]'),
917
+ ("petrol", '["amenity"="fuel"]'),
918
+ ("gas station", '["amenity"="fuel"]'),
919
+ ("cinema", '["amenity"="cinema"]'),
920
+ ("theatre", '["amenity"="theatre"]'),
921
+ ("travel", '["office"="travel_agent"]'),
922
+ ("hair", '["shop"="hairdresser"]'),
923
+ ("salon", '["shop"="beauty"]'),
924
+ ("beauty", '["shop"="beauty"]'),
925
+ ("ngo", '["office"="ngo"]'),
926
+ ("nonprofit", '["office"="ngo"]'),
927
+ ("food", '["amenity"~"restaurant|cafe|fast_food|bar"]'),
928
+ ("shop", '["shop"]'),
929
+ ("office", '["office"]'),
930
+ ("business", '["name"]'),
931
+ ]
932
+
933
+ # Try to find a specific match first
934
+ for keyword, tag_filter in _KEYWORD_MAP:
935
+ if keyword in combined:
936
+ return [tag_filter]
937
+
938
+ # Wildcard / generic — return anything with a name tag
939
+ if q in ("*", "", "all", "any"):
940
+ return ['["name"]']
941
+
942
+ # Fall back: name substring search (slow but correct for arbitrary queries)
943
+ safe_q = query.replace('"', "").replace("\\", "")
944
+ return [f'["name"~"{safe_q}",i]']
945
+
946
+
947
+ # ── HTTP helpers ──────────────────────────────────────────────────────────
948
+
949
+ async def _run_overpass(
950
+ self,
951
+ query: str,
952
+ category: str | None,
953
+ bbox: tuple[float, float, float, float],
954
+ limit: int,
955
+ ) -> list[Business]:
956
+ overpass_query = self._build_overpass_query(
957
+ query=query, category=category, bbox=bbox, limit=limit
958
+ )
959
+ data = await self._post_overpass(overpass_query)
960
+ elements = data.get("elements", [])
961
+
962
+ businesses: list[Business] = []
963
+ seen_names: set[str] = set()
964
+
965
+ for element in elements:
966
+ biz = _osm_element_to_business(element)
967
+ if biz is None:
968
+ continue
969
+ # Deduplicate by name+city within a single query
970
+ dedup_key = (biz.name.lower(), (biz.city or "").lower())
971
+ if dedup_key in seen_names:
972
+ continue
973
+ seen_names.add(dedup_key)
974
+ businesses.append(biz)
975
+ if len(businesses) >= limit:
976
+ break
977
+
978
+ return businesses
979
+
980
+ async def _post_overpass(self, query: str) -> dict:
981
+ """POST a query to the Overpass API and return parsed JSON."""
982
+ await self._rate_limit()
983
+ headers = {
984
+ "User-Agent": USER_AGENT,
985
+ "Accept": "application/json, */*",
986
+ "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
987
+ }
988
+
989
+ timeout = min(self.request_timeout if hasattr(self, "request_timeout") else REQUEST_TIMEOUT, 12.0)
990
+ # Try primary endpoint plus at most 1 mirror to keep response snappy before Nominatim fallback
991
+ endpoints = [self.endpoint]
992
+ for ep in self.fallback_endpoints:
993
+ if ep != self.endpoint and ep not in endpoints:
994
+ endpoints.append(ep)
995
+ if len(endpoints) >= 2:
996
+ break
997
+ last_exc: Exception | None = None
998
+
999
+ for ep in endpoints:
1000
+ try:
1001
+ async with httpx.AsyncClient(timeout=timeout) as client:
1002
+ resp = await client.post(
1003
+ ep,
1004
+ data={"data": query},
1005
+ headers=headers,
1006
+ )
1007
+ resp.raise_for_status()
1008
+ self._last_request = time.monotonic()
1009
+ return resp.json()
1010
+ except Exception as exc: # noqa: BLE001
1011
+ last_exc = exc
1012
+ logger.warning("Overpass endpoint '%s' failed: %s", ep, exc)
1013
+
1014
+ if last_exc:
1015
+ raise last_exc
1016
+ raise RuntimeError("No Overpass endpoints available")
1017
+
1018
+ async def _rate_limit(self) -> None:
1019
+ """Sleep if needed to respect the minimum Overpass request interval."""
1020
+ elapsed = time.monotonic() - self._last_request
1021
+ if elapsed < MIN_REQUEST_INTERVAL:
1022
+ await asyncio.sleep(MIN_REQUEST_INTERVAL - elapsed)
1023
+
1024
+ # ── Cache ─────────────────────────────────────────────────────────────────
1025
+
1026
+ def _get_cache(self, key: str) -> list[Business] | None:
1027
+ entry = self._cache.get(key)
1028
+ if entry is None:
1029
+ return None
1030
+ ts, data = entry
1031
+ if time.monotonic() - ts > self._cache_ttl:
1032
+ del self._cache[key]
1033
+ return None
1034
+ return data
1035
+
1036
+ def _set_cache(self, key: str, data: list[Business]) -> None:
1037
+ self._cache[key] = (time.monotonic(), data)
1038
+
1039
+ def clear_cache(self) -> None:
1040
+ self._cache.clear()