gmbscraper 0.2.0__tar.gz → 0.2.2__tar.gz

{gmbscraper-0.2.0 → gmbscraper-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gmbscraper
-Version: 0.2.0
+Version: 0.2.2
 Summary: Playwright-based scraper for Google Maps / Google Business Profile listings and reviews
 Project-URL: Homepage, https://github.com/exprtec/google-maps-scrapper
 Project-URL: Repository, https://github.com/exprtec/google-maps-scrapper
@@ -91,7 +91,14 @@ with google_maps_browser() as maps:
   reviews UI at all (throttled/limited view, or the reviews tab never opens),
   since a caller needs to distinguish "zero reviews" from "couldn't scrape."
   Its `GoogleLimitedViewError` subclass marks Google's signed-out limited
-  view, which signals throttling rather than a problem with one place.
+  view when no reviews could be read at all.
+- Signed-out Google Maps now usually serves a "limited view" whose reviews tab
+  stops after a few reviews behind a sign-in prompt. When `scrape_reviews` sees
+  it, it reads the place's name and feature id from Maps and opens the
+  "N Google reviews" panel from a Google Search for that name instead, which
+  lists every review. Only the panel whose link carries the same feature id is
+  used, so a similarly named business is never scraped by mistake. If Search
+  can't show the panel, it falls back to whatever the Maps tab shows.
 - Each place opens in a single page load, and each browser launch gets a
   random desktop Chrome user agent unless you pass `user_agent=`.
 - There's no retry/backoff or proxy rotation built in. This is a DOM scraper,

{gmbscraper-0.2.0 → gmbscraper-0.2.2}/README.md

@@ -67,7 +67,14 @@ with google_maps_browser() as maps:
   reviews UI at all (throttled/limited view, or the reviews tab never opens),
   since a caller needs to distinguish "zero reviews" from "couldn't scrape."
   Its `GoogleLimitedViewError` subclass marks Google's signed-out limited
-  view, which signals throttling rather than a problem with one place.
+  view when no reviews could be read at all.
+- Signed-out Google Maps now usually serves a "limited view" whose reviews tab
+  stops after a few reviews behind a sign-in prompt. When `scrape_reviews` sees
+  it, it reads the place's name and feature id from Maps and opens the
+  "N Google reviews" panel from a Google Search for that name instead, which
+  lists every review. Only the panel whose link carries the same feature id is
+  used, so a similarly named business is never scraped by mistake. If Search
+  can't show the panel, it falls back to whatever the Maps tab shows.
 - Each place opens in a single page load, and each browser launch gets a
   random desktop Chrome user agent unless you pass `user_agent=`.
 - There's no retry/backoff or proxy rotation built in. This is a DOM scraper,

{gmbscraper-0.2.0 → gmbscraper-0.2.2}/src/gmbscraper/_reviews.py

@@ -8,14 +8,16 @@ from typing import Any
 from gmbscraper._page import (
     click_reviews_tab,
     evaluate_json,
+    first_text,
     is_limited_maps_view,
     navigate_to_place,
     open_page,
     scroll_reviews_pane,
     set_reviews_sort,
 )
+from gmbscraper._search_reviews import place_feature_id, scrape_search_reviews_on_page
 from gmbscraper.models import MapsReview
-from gmbscraper.parsing import normalize_text, parse_relative_date
+from gmbscraper.parsing import extract_place_name_from_url, normalize_text, parse_relative_date
 
 EXTRACT_REVIEWS_JS = """() => {
     function extractUsername(card) {
@@ -152,6 +154,10 @@ EXTRACT_REVIEWS_JS = """() => {
 }"""
 
 
+# Scrolls that load no new review before the pane counts as fully read.
+IDLE_SCROLL_ROUNDS = 6
+
+
 class GoogleReviewsUnavailableError(RuntimeError):
     pass
 
@@ -201,8 +207,24 @@ def scrape_reviews_on_page(
 
     navigate_to_place(page, maps_url)
     if is_limited_maps_view(page):
-        raise GoogleLimitedViewError("Google Maps is showing a limited view — reviews unavailable")
+        # Signed-out Maps now caps the reviews tab at a few reviews behind a
+        # sign-in prompt; Search's review panel for the same place lists them all.
+        name = first_text(page, ("h1",)) or extract_place_name_from_url(page.url)
+        found = scrape_search_reviews_on_page(
+            page,
+            name=name,
+            feature_id=place_feature_id(page.url),
+            max_reviews=max_reviews,
+            known_ids=known_ids,
+        )
+        if found:
+            return found
+        navigate_to_place(page, maps_url)
     if not click_reviews_tab(page):
+        if is_limited_maps_view(page):
+            raise GoogleLimitedViewError(
+                "Google Maps is showing a limited view — reviews unavailable"
+            )
         raise GoogleReviewsUnavailableError("Could not open the Google Maps reviews tab")
 
     set_reviews_sort(page, label="Newest")
@@ -213,7 +235,10 @@ def scrape_reviews_on_page(
 
     seen: dict[str, MapsReview] = {}
     reached_known = False
+    # Rounds in a row that found no new review; growth is measured across a
+    # scroll, from one read of the pane to the next.
     idle_rounds = 0
+    read_count = 0
     max_scrolls = 80 if max_reviews is None else max(10, min(80, max_reviews // 2 + 8))
     for _ in range(1, max_scrolls + 1):
         for review in read_visible_reviews(page, limit=max_reviews):
@@ -229,15 +254,12 @@ def scrape_reviews_on_page(
             reached_known = reached_known or external_id in known_ids
         if reached_known or (max_reviews is not None and len(seen) >= max_reviews):
             break
-        before = len(seen)
+        idle_rounds = idle_rounds + 1 if len(seen) == read_count else 0
+        read_count = len(seen)
+        if idle_rounds >= IDLE_SCROLL_ROUNDS:
+            break
         if not scroll_reviews_pane(page):
             idle_rounds += 1
-        else:
-            idle_rounds = 0
-        if len(seen) == before:
-            idle_rounds += 1
-        if idle_rounds >= 6:
-            break
         time.sleep(0.5)
 
     if not seen:

gmbscraper-0.2.2/src/gmbscraper/_search_reviews.py

@@ -0,0 +1,188 @@
+"""Read a place's reviews from Google Search's review panel.
+
+Signed-out Google Maps now serves a "limited view" that shows only a handful of
+reviews and asks the visitor to sign in for the rest. The review panel behind a
+Search knowledge card (the "N Google reviews" link) still lists every review,
+so the Maps scraper falls back to it for limited-view pages.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import time
+from collections.abc import Collection
+from typing import Any
+from urllib.parse import quote_plus
+
+from gmbscraper._page import accept_consent, evaluate_json
+from gmbscraper.models import MapsReview
+from gmbscraper.parsing import PLACE_ID_PATTERN, normalize_text, parse_relative_date
+
+SEARCH_URL = "https://www.google.com/search?q={query}&hl={language}"
+REVIEW_CARD = "div.bwb7ce"
+REVIEW_PANE = "div.RVCQse"
+IDLE_SCROLL_ROUNDS = 4
+MAX_SCROLLS = 80
+
+EXTRACT_SEARCH_REVIEWS_JS = """() => {
+    // Filled stars are drawn in Google's gold; empty ones in a light grey.
+    function countStars(card) {
+        let filled = 0;
+        for (const path of card.querySelectorAll('.h3PQJ svg path')) {
+            const match = getComputedStyle(path).fill.match(/\\d+/g);
+            if (match && Number(match[0]) > 200 && Number(match[2]) < 120) filled += 1;
+        }
+        return filled;
+    }
+    function reviewText(card) {
+        // The full text sits in a hidden expanded copy; the visible one is truncated with "More".
+        const node = card.querySelector('.gbdaZe .d83Iyc') || card.querySelector('.d83Iyc');
+        if (!node) return '';
+        const copy = node.cloneNode(true);
+        copy.querySelectorAll('a[role="button"]').forEach((el) => el.remove());
+        return (copy.textContent || '').trim().replace(/^["\\u201c]|["\\u201d]$/g, '').trim();
+    }
+    return [...document.querySelectorAll('div.bwb7ce')].map((card) => ({
+        external_id: card.getAttribute('data-id')
+            || card.querySelector('[data-review-id]')?.getAttribute('data-review-id') || '',
+        username: (card.querySelector('.rhtdWc')?.textContent || '').trim(),
+        stars: countStars(card),
+        date_text: (card.querySelector('.y3Ibjb')?.textContent || '').trim(),
+        text: reviewText(card),
+    }));
+}"""
+
+
+def place_feature_id(url: str) -> str:
+    """The `0x…:0x…` feature id Google uses for a place in Maps and Search URLs."""
+    match = PLACE_ID_PATTERN.search(url or "")
+    return match.group(1).lower() if match else ""
+
+
+# Clicks are dispatched in the page: Search's layout often lays an overlay over
+# these controls, which makes a pointer click time out.
+CLICK_REVIEWS_LINK_JS = """(featureId) => {
+    const pattern = /^\\s*[\\d,.]+\\s+Google reviews?\\s*$/i;
+    const links = [...document.querySelectorAll('a[data-fid]')].filter((a) =>
+        a.getAttribute('data-fid').toLowerCase() === featureId && pattern.test(a.textContent || ''));
+    if (!links.length) return false;
+    links[0].scrollIntoView({block: 'center'});
+    links[0].click();
+    return true;
+}"""
+
+CLICK_NEWEST_JS = """() => {
+    const label = [...document.querySelectorAll('span, div, button')]
+        .find((el) => el.children.length === 0 && (el.textContent || '').trim() === 'Newest');
+    if (!label) return false;
+    (label.closest('[jsaction], [role="button"], [role="radio"], [role="tab"]') || label).click();
+    return true;
+}"""
+
+
+def open_search_review_panel(
+    page: Any, *, name: str, feature_id: str, language: str = "en"
+) -> bool:
+    """Open the review panel for `feature_id` from a Search for `name`. False if
+    Search doesn't show that place's review link (a name search can surface a
+    different business, so only a link carrying the same feature id counts)."""
+    if not feature_id:
+        return False
+    page.goto(
+        SEARCH_URL.format(query=quote_plus(name), language=language),
+        wait_until="domcontentloaded",
+        timeout=45000,
+    )
+    accept_consent(page)
+    with contextlib.suppress(Exception):
+        page.wait_for_selector(f'a[data-fid="{feature_id}"]', timeout=10000)
+    for _ in range(2):
+        try:
+            if not page.evaluate(CLICK_REVIEWS_LINK_JS, feature_id):
+                return False
+            page.wait_for_selector(REVIEW_CARD, timeout=15000)
+            return True
+        except Exception:
+            continue
+    return False
+
+
+def sort_search_reviews_newest(page: Any) -> bool:
+    try:
+        first_id = page.locator(REVIEW_CARD).first.get_attribute("data-id")
+        if not page.evaluate(CLICK_NEWEST_JS):
+            return False
+        # The list re-renders in place; wait for a different first card.
+        page.wait_for_function(
+            "([selector, id]) => { const c = document.querySelector(selector);"
+            " return c && c.getAttribute('data-id') !== id; }",
+            arg=[REVIEW_CARD, first_id],
+            timeout=8000,
+        )
+        page.wait_for_timeout(500)
+        return True
+    except Exception:
+        return False
+
+
+def read_search_reviews(page: Any) -> list[MapsReview]:
+    reviews: list[MapsReview] = []
+    for item in evaluate_json(page, EXTRACT_SEARCH_REVIEWS_JS):
+        stars = int(item.get("stars") or 0)
+        if not 1 <= stars <= 5:
+            continue
+        date_text = normalize_text(item.get("date_text", ""))
+        reviews.append(
+            MapsReview(
+                external_id=normalize_text(item.get("external_id", "")),
+                stars=stars,
+                username=normalize_text(item.get("username", "")),
+                text=normalize_text(item.get("text", "")),
+                date_text=date_text,
+                date=parse_relative_date(date_text),
+            )
+        )
+    return reviews
+
+
+def _scroll_pane(page: Any) -> None:
+    with contextlib.suppress(Exception):
+        page.evaluate(
+            "(selector) => { const el = document.querySelector(selector); if (el) el.scrollTop = el.scrollHeight; }",
+            REVIEW_PANE,
+        )
+
+
+def scrape_search_reviews_on_page(
+    page: Any,
+    *,
+    name: str,
+    feature_id: str,
+    max_reviews: int | None = None,
+    known_ids: Collection[str] = (),
+) -> list[MapsReview] | None:
+    """Reviews newest first, or None when Search can't show this place's review panel."""
+    if not name or not open_search_review_panel(page, name=name, feature_id=feature_id):
+        return None
+    sort_search_reviews_newest(page)
+
+    seen: dict[str, MapsReview] = {}
+    idle_rounds = 0
+    for _ in range(MAX_SCROLLS):
+        before = len(seen)
+        reached_known = False
+        for review in read_search_reviews(page):
+            external_id = review.resolved_external_id()
+            seen.setdefault(external_id, review)
+            reached_known = reached_known or external_id in known_ids
+        if reached_known or (max_reviews is not None and len(seen) >= max_reviews):
+            break
+        idle_rounds = idle_rounds + 1 if len(seen) == before else 0
+        if idle_rounds >= IDLE_SCROLL_ROUNDS:
+            break
+        _scroll_pane(page)
+        page.wait_for_timeout(900)
+        time.sleep(0.2)
+
+    reviews = list(seen.values())
+    return reviews if max_reviews is None else reviews[:max_reviews]

{gmbscraper-0.2.0 → gmbscraper-0.2.2}/src/gmbscraper/_version.py

@@ -4,4 +4,4 @@ Read by ``gmbscraper/__init__.py`` (public ``__version__``) and by the
 build backend (``pyproject.toml`` -> ``[tool.hatch.version]``).
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"

gmbscraper-0.2.2/tests/test_search_reviews.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from gmbscraper._search_reviews import place_feature_id
+
+
+def test_place_feature_id_reads_maps_place_urls() -> None:
+    url = (
+        "https://www.google.com/maps/place/Best+Builders/@31.44,73.17,17z/data=!3m1!4b1!4m6!3m5"
+        "!1s0x3922691029581c13:0x1B3FF9D4247B4509!8m2"
+    )
+    assert place_feature_id(url) == "0x3922691029581c13:0x1b3ff9d4247b4509"
+
+
+def test_place_feature_id_reads_search_review_fragments() -> None:
+    url = "https://www.google.com/search?q=x#lrd=0x3922691029581c13:0x1b3ff9d4247b4509,1,,,,"
+    assert place_feature_id(url) == "0x3922691029581c13:0x1b3ff9d4247b4509"
+
+
+def test_place_feature_id_is_empty_without_one() -> None:
+    assert place_feature_id("https://maps.google.com/?cid=1963562652097266953") == ""
+    assert place_feature_id("") == ""