gmbscraper 0.1.0__py3-none-any.whl

gmbscraper/__init__.py

@@ -0,0 +1,39 @@
+"""Playwright-based scraper for Google Maps / Google Business Profile listings and reviews."""
+
+from gmbscraper._matching import has_match_key, is_definitive_match, match_place, pick_best_match
+from gmbscraper._parallel import chunk_items, run_chunked
+from gmbscraper._reviews import GoogleReviewsUnavailableError, scrape_reviews
+from gmbscraper._scraper import GoogleMapsScraper, google_maps_browser
+from gmbscraper._version import __version__
+from gmbscraper.models import (
+    BusinessIdentity,
+    MapsReview,
+    MapsSearchLink,
+    MapsView,
+    OutletIdentity,
+    PlaceProfile,
+    SearchWindow,
+)
+from gmbscraper.parsing import normalize_text
+
+__all__ = [
+    "BusinessIdentity",
+    "GoogleMapsScraper",
+    "GoogleReviewsUnavailableError",
+    "MapsReview",
+    "MapsSearchLink",
+    "MapsView",
+    "OutletIdentity",
+    "PlaceProfile",
+    "SearchWindow",
+    "__version__",
+    "chunk_items",
+    "google_maps_browser",
+    "has_match_key",
+    "is_definitive_match",
+    "match_place",
+    "normalize_text",
+    "pick_best_match",
+    "run_chunked",
+    "scrape_reviews",
+]

gmbscraper/_adaptive_search.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import math
+import re
+from collections.abc import Callable
+from typing import Any, Protocol
+
+from gmbscraper.models import MapsSearchLink, MapsView, SearchWindow
+
+GOOGLE_MAPS_QUERY_RESULT_LIMIT = 120
+SATURATED_RESULT_COUNT = 110
+MAX_ADAPTIVE_DEPTH = 3
+MAX_ADAPTIVE_WINDOWS = 220
+MAX_SPLIT_ZOOM = 17
+MAP_VIEWPORT_WIDTH = 1366
+MAP_VIEWPORT_HEIGHT = 900
+FALLBACK_SEARCH_ZOOM = 12
+
+
+class SearchScraper(Protocol):
+    def new_page(self) -> Any: ...
+    def open_search(self, page: Any, query: str) -> None: ...
+    def open_search_at(
+        self, page: Any, query: str, *, lat: float, lng: float, zoom: int
+    ) -> None: ...
+    def collect_place_links(self, page: Any, limit: int) -> list[tuple[str, str]]: ...
+
+
+def collect_links_for_geo(
+    *,
+    scraper: SearchScraper,
+    query: str,
+    latitude: float | None = None,
+    longitude: float | None = None,
+    log: Callable[[str], None] | None = None,
+) -> list[MapsSearchLink]:
+    """Search a query and, if results saturate Google's ~120 cap, recursively
+    split the viewport into quadrants to pull results beyond that cap.
+    """
+    seed_links, seed_view = _collect_seed_links(scraper, query)
+    seen: set[str] = set()
+    links: list[MapsSearchLink] = []
+    _add_unique_links(links, seen, seed_links, keep_search_rank=True)
+    if log:
+        log(f"seed found={len(seed_links)} total={len(links)}")
+    if len(seed_links) < SATURATED_RESULT_COUNT:
+        return links
+
+    root_view = seed_view
+    if root_view is None and latitude is not None and longitude is not None:
+        root_view = MapsView(lat=latitude, lng=longitude, zoom=FALLBACK_SEARCH_ZOOM)
+    if root_view is None:
+        if log:
+            log("missing search bounds; seed query only")
+        return links
+
+    max_depth = _max_adaptive_depth(root_view.zoom)
+    queue: list[tuple[int, SearchWindow]] = [
+        (1, window)
+        for window in _split_window(
+            _search_window_from_view(root_view, label="root"),
+            next_zoom=root_view.zoom + 1,
+        )
+    ]
+    searched = 1
+    while queue and searched < MAX_ADAPTIVE_WINDOWS:
+        depth, window = queue.pop(0)
+        searched += 1
+        page = scraper.new_page()
+        try:
+            scraper.open_search_at(
+                page,
+                query,
+                lat=window.center_lat,
+                lng=window.center_lng,
+                zoom=window.zoom,
+            )
+            window_links = scraper.collect_place_links(page, GOOGLE_MAPS_QUERY_RESULT_LIMIT)
+        finally:
+            page.close()
+        added = _add_unique_links(links, seen, window_links, keep_search_rank=False)
+        if log:
+            log(
+                f"adaptive d{depth} z{window.zoom} {window.label} "
+                f"found={len(window_links)} new={added} total={len(links)}"
+            )
+        if _should_split_window(window_links, depth=depth, zoom=window.zoom, max_depth=max_depth):
+            queue.extend(
+                (depth + 1, child) for child in _split_window(window, next_zoom=window.zoom + 1)
+            )
+    return links
+
+
+def _max_adaptive_depth(root_zoom: int) -> int:
+    zoom_headroom = max(0, MAX_SPLIT_ZOOM - root_zoom)
+    return min(zoom_headroom, MAX_ADAPTIVE_DEPTH)
+
+
+def _should_split_window(
+    window_links: list[tuple[str, str]],
+    *,
+    depth: int,
+    zoom: int,
+    max_depth: int,
+) -> bool:
+    if len(window_links) < SATURATED_RESULT_COUNT:
+        return False
+    if depth >= max_depth:
+        return False
+    return zoom < MAX_SPLIT_ZOOM
+
+
+def _collect_seed_links(
+    scraper: SearchScraper, query: str
+) -> tuple[list[tuple[str, str]], MapsView | None]:
+    page = scraper.new_page()
+    try:
+        scraper.open_search(page, query)
+        links = scraper.collect_place_links(page, GOOGLE_MAPS_QUERY_RESULT_LIMIT)
+        return links, _parse_maps_view(page.url)
+    finally:
+        page.close()
+
+
+def _add_unique_links(
+    links: list[MapsSearchLink],
+    seen: set[str],
+    new_links: list[tuple[str, str]],
+    *,
+    keep_search_rank: bool,
+) -> int:
+    added = 0
+    for search_position, (maps_url, fallback_name) in enumerate(new_links, start=1):
+        if maps_url in seen:
+            continue
+        seen.add(maps_url)
+        links.append(
+            MapsSearchLink(
+                maps_url=maps_url,
+                fallback_name=fallback_name,
+                search_rank=search_position if keep_search_rank else None,
+            )
+        )
+        added += 1
+    return added
+
+
+def _parse_maps_view(url: str) -> MapsView | None:
+    match = re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?),(\d+(?:\.\d+)?)z", url)
+    if match is None:
+        return None
+    return MapsView(
+        lat=float(match.group(1)),
+        lng=float(match.group(2)),
+        zoom=max(1, round(float(match.group(3)))),
+    )
+
+
+def _search_window_from_view(view: MapsView, *, label: str) -> SearchWindow:
+    lat_span, lng_span = _viewport_span_degrees(view.lat, view.zoom)
+    return SearchWindow(
+        south=max(-85.0, view.lat - lat_span / 2),
+        west=_normalize_lng(view.lng - lng_span / 2),
+        north=min(85.0, view.lat + lat_span / 2),
+        east=_normalize_lng(view.lng + lng_span / 2),
+        zoom=view.zoom,
+        label=label,
+    )
+
+
+def _viewport_span_degrees(center_lat: float, zoom: int) -> tuple[float, float]:
+    world_pixels = 256 * (2**zoom)
+    lat_rad = math.radians(_clamp(center_lat, -85.0, 85.0))
+    meters_per_pixel = 156543.03392 * math.cos(lat_rad) / (2**zoom)
+    width_m = MAP_VIEWPORT_WIDTH * meters_per_pixel
+    height_m = MAP_VIEWPORT_HEIGHT * meters_per_pixel
+    lat_span = height_m / 110_574
+    lng_span = width_m / max(1.0, 111_320 * math.cos(lat_rad))
+    if world_pixels <= 0:
+        return 0.0, 0.0
+    return max(lat_span, 0.001), max(lng_span, 0.001)
+
+
+def _split_window(window: SearchWindow, *, next_zoom: int) -> list[SearchWindow]:
+    mid_lat = (window.south + window.north) / 2
+    mid_lng = (window.west + window.east) / 2
+    return [
+        SearchWindow(window.south, window.west, mid_lat, mid_lng, next_zoom, f"{window.label}.sw"),
+        SearchWindow(window.south, mid_lng, mid_lat, window.east, next_zoom, f"{window.label}.se"),
+        SearchWindow(mid_lat, window.west, window.north, mid_lng, next_zoom, f"{window.label}.nw"),
+        SearchWindow(mid_lat, mid_lng, window.north, window.east, next_zoom, f"{window.label}.ne"),
+    ]
+
+
+def _normalize_lng(value: float) -> float:
+    while value < -180:
+        value += 360
+    while value > 180:
+        value -= 360
+    return value
+
+
+def _clamp(value: float, lower: float, upper: float) -> float:
+    return min(max(value, lower), upper)

gmbscraper/_matching.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from gmbscraper.models import BusinessIdentity, OutletIdentity, PlaceProfile
+from gmbscraper.parsing import normalize_domain, normalize_name, normalize_phone
+
+_REASON_RANK = {
+    "phone_and_domain_match": 0,
+    "phone_match": 1,
+    "domain_match": 2,
+    "exact_name_match": 3,
+}
+
+
+def has_match_key(outlet: OutletIdentity, business: BusinessIdentity) -> bool:
+    """Whether there's enough identity signal to attempt a match at all."""
+    return bool(
+        normalize_phone(outlet.phone)
+        or normalize_domain(business.website)
+        or normalize_name(business.name)
+        or normalize_name(outlet.name)
+    )
+
+
+def match_place(
+    place: PlaceProfile,
+    outlet: OutletIdentity,
+    business: BusinessIdentity,
+) -> PlaceProfile | None:
+    """Score a scraped place against known identity, tagging `place.match_reason` on success."""
+    outlet_phone = normalize_phone(outlet.phone)
+    place_phone = normalize_phone(place.phone)
+    business_domain = normalize_domain(business.website)
+    place_domain = normalize_domain(place.website)
+    place_name = normalize_name(place.name)
+    business_name = normalize_name(business.name)
+    outlet_name = normalize_name(outlet.name)
+    phone_match = bool(outlet_phone and place_phone and outlet_phone == place_phone)
+    domain_match = bool(business_domain and place_domain and business_domain == place_domain)
+    exact_name_match = bool(place_name and place_name in {business_name, outlet_name})
+    if not phone_match and not domain_match and not exact_name_match:
+        return None
+    if phone_match and domain_match:
+        place.match_reason = "phone_and_domain_match"
+    elif phone_match:
+        place.match_reason = "phone_match"
+    elif domain_match:
+        place.match_reason = "domain_match"
+    else:
+        place.match_reason = "exact_name_match"
+    return place
+
+
+def is_definitive_match(place: PlaceProfile) -> bool:
+    """True if the match reason is strong enough to stop searching further candidates."""
+    return place.match_reason in _REASON_RANK
+
+
+def pick_best_match(places: list[PlaceProfile]) -> PlaceProfile | None:
+    if not places:
+        return None
+    return sorted(
+        places,
+        key=lambda item: (_REASON_RANK.get(item.match_reason or "", 99), item.rank),
+    )[0]