google-maps-scraper 0.1.0__py3-none-any.whl

gmaps_scraper/parser.py

@@ -0,0 +1,546 @@
+"""DOM parsing logic for extracting place details from Google Maps pages."""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from typing import Optional
+
+from playwright.async_api import Page
+
+from gmaps_scraper.models import PlaceDetails, Review
+from gmaps_scraper.utils import (
+    extract_coordinates_from_url,
+    parse_review_count,
+    safe_float,
+    safe_int,
+)
+
+logger = logging.getLogger("gmaps_scraper")
+
+
+async def parse_place_details(page: Page) -> PlaceDetails:
+    """Extract standard place details using playwright page."""
+    details = PlaceDetails()
+    details.url = page.url
+
+    t0 = time.time()
+
+    # Try to expand hours if possible before evaluation
+    try:
+        hours_button = await page.query_selector(
+            'div[data-hide-tooltip-on-mouse-move] button[data-item-id*="oh"],'
+            'button[aria-label*="hour"], button[aria-label*="營業"], button[aria-label*="営業"]'
+        )
+        if hours_button:
+            await hours_button.click(timeout=1000)
+            await page.wait_for_timeout(500)
+    except Exception:
+        pass
+
+    # We use a single evaluate call to run all DOM queries entirely in the browser.
+    # This prevents the massive async CDP overhead from hundreds of Python queries.
+    # NOTE: All regex patterns here use [0-9] instead of \d, and [(] [)] instead of
+    # \( \) to avoid Python raw-string / JS double-escape confusion.
+    js_extractor = r"""() => {
+        const result = {};
+        
+        // Find the main place panel to scope our search and avoid "Nearby Places" bleed
+        let panel = document.body;
+        let titleEl = document.querySelector('h1.DUwDvf');
+        if (titleEl) {
+            let curr = titleEl;
+            for(let i=0; i<8; i++) {
+                if(curr.parentElement && curr.parentElement.tagName !== 'BODY') {
+                    curr = curr.parentElement;
+                }
+            }
+            panel = curr;
+        }
+        
+        // Helper: first visible text from selector list (no digit requirement)
+        const firstText = (selectors) => {
+            for (let sel of selectors) {
+                let el = panel.querySelector(sel);
+                if (el && el.innerText && el.innerText.trim()) {
+                    return el.innerText.trim();
+                }
+            }
+            return null;
+        };
+        
+        // Helper: first visible text containing at least one digit
+        const firstTextWithNumber = (selectors) => {
+            for (let sel of selectors) {
+                let el = panel.querySelector(sel);
+                if (el && el.innerText && el.innerText.trim() && el.innerText.match(/[0-9]/)) {
+                    return el.innerText.trim();
+                }
+            }
+            return null;
+        };
+
+        // Helper: data-item-id wrapper
+        const dataItemId = (keywords) => {
+            for (let kw of keywords) {
+                let kw_l = kw.toLowerCase().replace(/ /g, '');
+                let el = document.querySelector(`button[data-item-id="${kw_l}"]`);
+                if (el && el.innerText && el.innerText.trim()) {
+                    let text = el.innerText.trim();
+                    for (let k of keywords) {
+                        if (text.startsWith(k)) return text.substring(k.length).replace(/^:/,'').trim();
+                    }
+                    return text;
+                }
+            }
+            return null;
+        };
+        
+        // ===== Name =====
+        result.name = titleEl ? titleEl.innerText.trim() : "";
+        if (!result.name) {
+            result.name = firstText([
+                'h1.DUwDvf',
+                'h2[data-attrid="title"]',
+                '.x3AX1-LfntMc-header-title-title',
+                'h1.lfPIob', 'h1[class*="header"] span',
+                'h1.fontHeadlineLarge', 'div[role="main"] h1', 'h1'
+            ]);
+        }
+        
+        // ===== Rating =====
+        result.rating = firstTextWithNumber([
+            'div.F7nice > span > span[aria-hidden="true"]:first-child',
+            'div.fontDisplayLarge',
+            'span[role="img"][aria-label*="star"]',
+            'span.ceNzKf[role="img"]',
+        ]);
+        if (!result.rating) {
+            // Fallback: check aria-label on star icons
+            let starEls = panel.querySelectorAll('span[role="img"]');
+            for (let el of starEls) {
+                let aria = el.getAttribute('aria-label');
+                if (aria && (aria.includes('star') || aria.includes('顆星') || aria.includes('つ星'))) {
+                    let m = aria.match(/([0-9]+[.,]?[0-9]*)/);
+                    if (m) { result.rating = m[1]; break; }
+                }
+            }
+        }
+        
+        // ===== Review Count (PRIMARY: aria-label on review button) =====
+        // Strategy: use aria-label which is the most reliable source across all locales.
+        // Pattern: "1,296 件の Google クチコミ" or "1,296 reviews" or "1,296 篇 Google 評論"
+        // IMPORTANT: Collect ALL candidates and pick the LARGEST, because tag/filter
+        // buttons (e.g. "晴空塔 3,175則評論") can appear before the main total count.
+        const reviewKeywords = ['review', '評論', 'クチコミ'];
+        const reviewCountRegex = /([0-9][0-9,.]*[ ]*[KM万萬]?)[ ]*(?:reviews?|則[ ]*Google[ ]*評論|篇[ ]*Google[ ]*評論|則評論|篇評論|評論|件の[ ]*Google[ ]*クチコミ|件のクチコミ|クチコミ)/i;
+        
+        // Helper: parse a count string like "93,962" or "3.6K" to a numeric value for comparison
+        const parseCount = (s) => {
+            s = s.trim().toUpperCase();
+            let mul = 1;
+            if (s.includes('K')) mul = 1000;
+            else if (s.includes('M')) mul = 1000000;
+            else if (s.includes('萬') || s.includes('万')) mul = 10000;
+            s = s.replace(/[KM萬万]/g, '');
+            let n = parseFloat(s.replace(/,/g, ''));
+            return isNaN(n) ? 0 : n * mul;
+        };
+        
+        // Method 1: aria-label on buttons — collect ALL and pick largest
+        let bestCount = null;
+        let bestCountVal = 0;
+        let bestSource = null;
+        
+        // Additional regex: "更多評論 (93,959)" / "More reviews (93,959)" — keyword BEFORE number
+        const reverseCountRegex = /(?:reviews?|評論|クチコミ)\s*[(]([0-9][0-9,.]*[ ]*[KM万萬]*)[)]/i;
+        
+        let allButtons = panel.querySelectorAll('button');
+        for (let btn of allButtons) {
+            let aria = btn.getAttribute('aria-label');
+            if (aria && reviewKeywords.some(kw => aria.includes(kw))) {
+                let m = aria.match(reviewCountRegex) || aria.match(reverseCountRegex);
+                if (m) {
+                    let val = parseCount(m[1]);
+                    if (val > bestCountVal) {
+                        bestCountVal = val;
+                        bestCount = m[1].trim();
+                        bestSource = 'aria-button';
+                    }
+                }
+            }
+        }
+        
+        // Method 2: aria-label on any element — also collect and compare
+        if (!bestCount) {
+            let allEls = panel.querySelectorAll('*');
+            for (let el of allEls) {
+                let aria = el.getAttribute('aria-label');
+                if (aria && aria.match(/[0-9]/) && reviewKeywords.some(kw => aria.includes(kw))) {
+                    let m = aria.match(reviewCountRegex) || aria.match(reverseCountRegex);
+                    if (m) {
+                        let val = parseCount(m[1]);
+                        if (val > bestCountVal) {
+                            bestCountVal = val;
+                            bestCount = m[1].trim();
+                            bestSource = 'aria-any';
+                        }
+                    }
+                }
+            }
+        }
+        
+        // Method 3: innerText of F7nice review span (displayed count next to rating)
+        // Always runs — F7nice is the most authoritative source for total review count.
+        // It overrides tag-button counts which are per-filter, not total.
+        {
+            let reviewSpans = panel.querySelectorAll('div.F7nice span');
+            for (let sp of reviewSpans) {
+                let text = sp.innerText ? sp.innerText.trim() : '';
+                // Match "(1,296)" or "1,296" — parenthesized or bare number
+                let m = text.match(/^[(]?([0-9][0-9,.]*[ ]*[KM万萬]*)[)]?$/i);
+                if (m && !text.match(/^[0-9]+[.,][0-9]+$/) /* skip rating like 4.2 */) {
+                    let val = parseCount(m[1]);
+                    if (val > bestCountVal) {
+                        bestCountVal = val;
+                        bestCount = m[1].trim();
+                        bestSource = 'f7nice-span';
+                    }
+                }
+            }
+        }
+        
+        if (bestCount) {
+            result.review_count = bestCount;
+            result._rc_source = bestSource;
+        }
+        
+        // Method 4: Text walker — match "N reviews" / "N 件のクチコミ" / "N 篇評論"
+        if (!result.review_count) {
+            let walk = document.createTreeWalker(panel, NodeFilter.SHOW_TEXT, null, false);
+            let n;
+            while (n = walk.nextNode()) {
+                let text = n.nodeValue.trim();
+                if (text.length > 0 && text.length < 100) {
+                    let m = text.match(reviewCountRegex);
+                    if (m) {
+                        result.review_count = m[1].trim();
+                        result._rc_source = 'text-walker';
+                        break;
+                    }
+                }
+            }
+        }
+        
+        // Method 5: Tab buttons — "Reviews" tab with count
+        if (!result.review_count) {
+            let tabs = panel.querySelectorAll('div[role="tablist"] button');
+            for (let tab of tabs) {
+                let text = tab.innerText || "";
+                if (reviewKeywords.some(kw => text.toLowerCase().includes(kw.toLowerCase()))) {
+                    let m = text.match(/([0-9][0-9,.]*[ ]*[KM万萬]*)/i);
+                    if (m) {
+                        result.review_count = m[1].trim();
+                        result._rc_source = 'tab';
+                        break;
+                    }
+                }
+            }
+        }
+        
+        // Method 6: Full body text search (for hotel/lodging pages where the scraper
+        // has clicked the star icon to open the Reviews summary panel)
+        // NOTE: Using \s* not [ ]* because innerText has \n between DOM elements
+        if (!result.review_count) {
+            let bodyText = document.body.innerText;
+            let m = bodyText.match(/([0-9][0-9,.]*)\s*(?:件の|件\s*の)?\s*(?:Google\s*)?(?:クチコミ|reviews?|評論)/i);
+            if (m) {
+                result.review_count = m[1].trim();
+                result._rc_source = 'body-text';
+            }
+        }
+
+        // ===== Category =====
+        result.category = firstText([
+            'button[jsaction*="category"]',
+            'span[jstcache] button.DkEaL',
+            'div[role="main"] button.DkEaL'
+        ]);
+        
+        // ===== Address =====
+        result.address = dataItemId(["地址", "Address", "住所", "所在地"]);
+        if (!result.address) {
+            let addressBtn = document.querySelector('button[data-item-id="address"]');
+            if (addressBtn) result.address = addressBtn.innerText.trim();
+        }
+        if (!result.address) {
+            // Fallback: look for aria-label containing address keywords
+            let btns = document.querySelectorAll('button[aria-label]');
+            for (let btn of btns) {
+                let aria = btn.getAttribute('aria-label') || '';
+                if (["地址", "Address", "住所"].some(kw => aria.includes(kw))) {
+                    let text = aria;
+                    ["地址", "Address", "住所"].forEach(kw => {
+                        text = text.replace(new RegExp(`^${kw}:?`, 'i'), '').trim();
+                    });
+                    if (text) { result.address = text; break; }
+                }
+            }
+        }
+        // Clean newlines from address (multi-line DOM elements)
+        if (result.address) {
+            result.address = result.address.replace(/[\r\n]+/g, ' ').replace(/\s{2,}/g, ' ').trim();
+        }
+        
+        // ===== Phone =====
+        result.phone = dataItemId(["電話", "Phone", "電話番号"]);
+        if (!result.phone) {
+            let phoneEl = document.querySelector('button[data-item-id^="phone:"]');
+            if (phoneEl) result.phone = phoneEl.innerText.trim();
+        }
+        
+        // ===== Website =====
+        let webEl = document.querySelector('a[data-item-id="authority"]');
+        if (webEl) result.website = webEl.getAttribute('href');
+        if (!result.website) result.website = dataItemId(["網站", "Website", "ウェブサイト"]);
+        
+        // ===== Plus Code =====
+        result.plus_code = dataItemId(["Plus Code", "Plus code", "プラスコード"]);
+        if (!result.plus_code) {
+            let plusEl = document.querySelector('button[data-item-id="oloc"]');
+            if (plusEl) result.plus_code = plusEl.innerText.trim();
+        }
+        
+        // ===== Closed Status =====
+        let closedEl = panel.querySelector('span[class*="permanently"], div[class*="closed"]');
+        if (closedEl && closedEl.innerText) {
+            let lower = closedEl.innerText.toLowerCase();
+            if (lower.includes("permanently") || lower.includes("永久")) result.permanently_closed = true;
+            else if (lower.includes("temporarily") || lower.includes("暫時")) result.temporarily_closed = true;
+        }
+
+        // ===== Description =====
+        result.description = firstText(['div[class*="editorial"] span', 'div[class*="description"] span']);
+        
+        // ===== Hours =====
+        result.hours = [];
+        let hrsRows = document.querySelectorAll('table[class*="eK4R0e"] tr, table[class*="hours"] tr');
+        if (hrsRows.length > 0) {
+            for (let row of hrsRows) {
+                if (row.innerText && row.innerText.trim()) result.hours.push(row.innerText.trim());
+            }
+        }
+        if (result.hours.length === 0) {
+            let hoursBtn = document.querySelector('div[data-hide-tooltip-on-mouse-move] button[data-item-id*="oh"], button[aria-label*="hour"], button[aria-label*="營業"], button[aria-label*="営業"]');
+            if (hoursBtn) {
+                let aria = hoursBtn.getAttribute('aria-label');
+                if (aria) result.hours.push(aria);
+            }
+        }
+        
+        // ===== Photos Count =====
+        let photoBtn = panel.querySelector('button[aria-label*="photo"], button[aria-label*="張相片"], button[aria-label*="写真"], button[aria-label*="枚の写真"]');
+        if (photoBtn) {
+            let aria = photoBtn.getAttribute('aria-label') || '';
+            let m = aria.match(/([0-9][0-9,.]*)/);
+            if (m) result.photos_count = m[1].replace(/,/g, '');
+        }
+
+        return result;
+    }"""
+
+    extracted = await page.evaluate(js_extractor)
+
+    # Populate python model
+    details.name = extracted.get("name")
+    if extracted.get("rating"):
+        details.rating = safe_float(extracted["rating"].replace(",", "."))
+    if extracted.get("review_count"):
+        c = extracted["review_count"]
+        m = re.search(r"([0-9,.]+[\sKM万萬]*)", c, re.IGNORECASE)
+        if m:
+            details.review_count = parse_review_count(m.group(1))
+        else:
+            details.review_count = parse_review_count(c)
+
+    details.address = extracted.get("address")
+    details.phone = extracted.get("phone")
+    details.website = extracted.get("website")
+    cat = extracted.get("category")
+    if cat:
+        # Strip leading/trailing separators (·, ‧, ·, •) and whitespace
+        details.category = re.sub(r'^[\s·‧·•·]+|[\s·‧·•·]+$', '', cat).strip()
+    else:
+        details.category = None
+    hours_list = extracted.get("hours", [])
+    if hours_list:
+        details.hours = hours_list
+    details.plus_code = extracted.get("plus_code")
+    details.description = extracted.get("description")
+    
+    if extracted.get("photos_count"):
+        details.photos_count = safe_int(extracted["photos_count"])
+
+    if extracted.get("permanently_closed"):
+        details.permanently_closed = True
+    elif extracted.get("temporarily_closed"):
+        details.temporarily_closed = True
+
+    # Capture resolved Google Maps URL (after redirect from search URL)
+    current_url = page.url
+    if '/maps/' in current_url:
+        details.google_maps_url = current_url
+
+    # Extract coordinates from URL
+    coords = extract_coordinates_from_url(current_url)
+    if coords:
+        details.latitude, details.longitude = coords
+
+    # Try place_id from URL
+    place_id_match = re.search(r"place_id[=:]([A-Za-z0-9_-]+)", page.url)
+    if place_id_match:
+        details.place_id = place_id_match.group(1)
+
+    if not details.place_id:
+        place_id_match = re.search(r"0x[0-9a-f]+:0x[0-9a-f]+", page.url)
+        if place_id_match:
+            details.place_id = place_id_match.group(0)
+
+    # FALLBACK 1: Category from span.mgr77e (hotel/lodging category label)
+    if details.category is None:
+        try:
+            cat_js = r"""() => {
+                let el = document.querySelector('span.mgr77e');
+                return el ? el.innerText.trim() : null;
+            }"""
+            cat_value = await page.evaluate(cat_js)
+            if cat_value:
+                details.category = cat_value.strip('·').strip()
+        except Exception:
+            pass
+
+    # FALLBACK 3: If still no review count, try HTML source regex
+    if details.review_count is None:
+        try:
+            html = await page.content()
+            # Search for review count patterns in raw HTML/JSON data embedded by Google
+            for kw in [
+                "件の Google クチコミ", "件のクチコミ", "クチコミ",
+                "reviews", "review",
+                "則 Google 評論", "篇 Google 評論", "則評論", "篇評論", "評論",
+            ]:
+                idx = html.find(kw)
+                if idx != -1:
+                    chunk = html[max(0, idx - 20):idx].strip()
+                    num_match = re.search(r'([0-9][0-9,.]{0,10}[\sKM万萬]?)\s*$', chunk)
+                    if num_match:
+                        details.review_count = parse_review_count(num_match.group(1).strip())
+                        logger.debug("Review count found via HTML fallback: %s", details.review_count)
+                        break
+        except Exception as e:
+            logger.debug("HTML fallback failed: %s", e)
+
+    t1 = time.time()
+    logger.info("DOM parsing took %.2fs (review_count=%s, source=%s)",
+                t1 - t0, details.review_count, extracted.get("_rc_source", "fallback"))
+
+    return details
+
+
+async def parse_reviews(page: Page, max_reviews: int = 20) -> list[Review]:
+    """Extract reviews from a Google Maps place page."""
+    reviews: list[Review] = []
+
+    # Try to click the reviews tab/button
+    try:
+        review_tab = await page.query_selector(
+            'button[aria-label*="review"], '
+            'button[aria-label*="評論"], '
+            'button[aria-label*="クチコミ"], '
+            'div[role="tablist"] button:nth-child(2)'
+        )
+        if review_tab:
+            await review_tab.click(timeout=2000)
+            await page.wait_for_timeout(2000)
+    except Exception as e:
+        logger.warning("Failed to click review tab: %s", e)
+        return reviews
+
+    # Scroll the reviews panel to load more reviews
+    try:
+        scrollable = await page.query_selector(
+            'div[role="main"] div.m6QErb.DxyBCb.kA9KIf.dS8AEf'
+        )
+        if not scrollable:
+            scrollable = await page.query_selector('div[role="main"]')
+
+        if scrollable:
+            for _ in range(min(max_reviews // 5, 20)):
+                await scrollable.evaluate("el => el.scrollTop = el.scrollHeight")
+                await page.wait_for_timeout(1000)
+                # Check if we have enough reviews loaded
+                review_elements = await page.evaluate(
+                    """() => document.querySelectorAll('div[data-review-id], div[class*="review"]').length"""
+                )
+                if review_elements >= max_reviews:
+                    break
+    except Exception:
+        pass
+
+    # Extract all reviews via page evaluate
+    js_reviews = r"""(max_len) => {
+        let results = [];
+        // Click all "more" buttons first
+        let moreBtns = document.querySelectorAll('button[aria-label*="See more"], button[aria-label*="もっと見る"], button.w8nwRe.kyuRq');
+        for (let i=0; i<moreBtns.length && i<max_len; i++) {
+            try { moreBtns[i].click() } catch(e){}
+        }
+
+        let containers = document.querySelectorAll('div[data-review-id]');
+        if (containers.length === 0) containers = document.querySelectorAll('div[class*="jftiEf"]');
+
+        for (let i=0; i<containers.length && i<max_len; i++) {
+            let container = containers[i];
+            let rev = {};
+            
+            // Name
+            let nameEl = container.querySelector('div.d4r55, button[class*="reviewer"] div, a[class*="name"]');
+            if (nameEl) rev.author = nameEl.innerText.trim();
+            
+            // Text 
+            let textEl = container.querySelector('span[class*="wiI7pd"], div.MyEned span');
+            if (textEl) rev.text = textEl.innerText.trim();
+            
+            // Time
+            let timeEl = container.querySelector('span[class*="rsqaWe"], span.dehysf');
+            if (timeEl) rev.time = timeEl.innerText.trim();
+
+            // Stars
+            let starEl = container.querySelector('span[role="img"][aria-label*="star"], span[role="img"][aria-label*="顆星"], span[role="img"][aria-label*="つ星"]');
+            if (starEl) {
+                let aria = starEl.getAttribute('aria-label');
+                if (aria){
+                    let m = aria.match(/([0-9])/);
+                    if (m) rev.rating = parseInt(m[1]);
+                }
+            }
+            results.push(rev);
+        }
+        return results;
+    }"""
+
+    # Brief wait for "more" button clicks to settle
+    await page.wait_for_timeout(500)
+    extracted_reviews = await page.evaluate(js_reviews, max_reviews)
+
+    for r in extracted_reviews:
+        rev = Review()
+        rev.author = r.get("author")
+        rev.text = r.get("text")
+        rev.time = r.get("time")
+        rev.rating = r.get("rating")
+        if rev.author or rev.text:
+            reviews.append(rev)
+
+    return reviews