google-maps-scraper 0.1.0__py3-none-any.whl

gmaps_scraper/__init__.py

@@ -0,0 +1,33 @@
+"""Google Maps Reviews Scraper - Scrape place details without API key.
+
+Usage:
+    # Async API
+    async with GoogleMapsScraper() as scraper:
+        result = await scraper.scrape("https://www.google.com/maps/search/...")
+        print(result.place.name, result.place.rating)
+
+    # Sync convenience
+    from gmaps_scraper import scrape_place
+    result = scrape_place("https://www.google.com/maps/search/...")
+
+    # Batch processing
+    from gmaps_scraper import scrape_batch, ScrapeConfig
+    config = ScrapeConfig(concurrency=5)
+    results = asyncio.run(scrape_batch(urls, config))
+"""
+
+from gmaps_scraper.batch import scrape_batch
+from gmaps_scraper.models import PlaceDetails, Review, ScrapeConfig, ScrapeResult
+from gmaps_scraper.scraper import GoogleMapsScraper, scrape_place
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "GoogleMapsScraper",
+    "scrape_place",
+    "scrape_batch",
+    "PlaceDetails",
+    "Review",
+    "ScrapeConfig",
+    "ScrapeResult",
+]

gmaps_scraper/batch.py

@@ -0,0 +1,263 @@
+"""Async batch processor for high-throughput Google Maps scraping."""
+
+from __future__ import annotations
+
+import asyncio
+import csv
+import json
+import logging
+import random
+from datetime import datetime
+from pathlib import Path
+from typing import Callable, Optional
+
+from gmaps_scraper.models import ScrapeConfig, ScrapeResult
+from gmaps_scraper.scraper import GoogleMapsScraper
+
+logger = logging.getLogger("gmaps_scraper")
+
+
+async def scrape_batch(
+    urls: list[str],
+    config: ScrapeConfig | None = None,
+    output_path: str | Path | None = None,
+    on_result: Optional[Callable[[ScrapeResult, int, int], None]] = None,
+    resume: bool = True,
+) -> list[ScrapeResult]:
+    """Scrape a batch of Google Maps URLs concurrently.
+
+    Args:
+        urls: List of Google Maps URLs to scrape.
+        config: Scraper configuration (concurrency, delays, etc.).
+        output_path: Path to save results incrementally (JSON or CSV).
+        on_result: Callback called after each result (result, index, total).
+        resume: If True, skip URLs already present in the output file.
+
+    Returns:
+        List of ScrapeResult objects.
+
+    Example:
+        >>> import asyncio
+        >>> from gmaps_scraper import scrape_batch, ScrapeConfig
+        >>> urls = ["https://www.google.com/maps/search/...", ...]
+        >>> config = ScrapeConfig(concurrency=5, headless=True)
+        >>> results = asyncio.run(scrape_batch(urls, config))
+    """
+    config = config or ScrapeConfig()
+    output_path = Path(output_path) if output_path else None
+
+    # Load already scraped URLs for resume
+    completed_urls: set[str] = set()
+    existing_results: list[ScrapeResult] = []
+    if resume and output_path and output_path.exists():
+        existing_results, completed_urls = _load_existing_results(output_path)
+        if completed_urls:
+            logger.info(
+                "Resuming: %d URLs already scraped, %d remaining",
+                len(completed_urls),
+                len(urls) - len(completed_urls),
+            )
+
+    # Filter out already completed URLs
+    remaining_urls = [u for u in urls if u.strip() not in completed_urls]
+    total = len(urls)
+    completed_count = len(completed_urls)
+
+    if not remaining_urls:
+        logger.info("All %d URLs already scraped!", total)
+        return existing_results
+
+    results: list[ScrapeResult] = list(existing_results)
+    semaphore = asyncio.Semaphore(config.concurrency)
+    save_lock = asyncio.Lock()
+
+    async with GoogleMapsScraper(config) as scraper:
+
+        async def _scrape_one(url: str) -> ScrapeResult:
+            nonlocal completed_count
+            async with semaphore:
+                # Random delay to avoid rate limiting
+                delay = random.uniform(config.delay_min, config.delay_max)
+                await asyncio.sleep(delay)
+
+                try:
+                    result = await scraper.scrape(url)
+                except Exception as e:
+                    result = ScrapeResult(
+                        input_url=url,
+                        success=False,
+                        error=str(e),
+                    )
+
+                completed_count += 1
+
+                # Callback
+                if on_result:
+                    try:
+                        on_result(result, completed_count, total)
+                    except Exception:
+                        pass
+
+                # Periodic save
+                async with save_lock:
+                    results.append(result)
+                    if (
+                        output_path
+                        and completed_count % config.save_interval == 0
+                    ):
+                        _save_results(results, output_path)
+                        logger.info(
+                            "Progress saved: %d/%d (%.1f%%)",
+                            completed_count,
+                            total,
+                            completed_count / total * 100,
+                        )
+
+                return result
+
+        # Run all tasks with concurrency control
+        tasks = [_scrape_one(url) for url in remaining_urls]
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    # Final save
+    if output_path:
+        _save_results(results, output_path)
+        logger.info("Final results saved to %s", output_path)
+
+    return results
+
+
+def _load_existing_results(
+    path: Path,
+) -> tuple[list[ScrapeResult], set[str]]:
+    """Load existing results for resume capability."""
+    results: list[ScrapeResult] = []
+    urls: set[str] = set()
+
+    try:
+        if path.suffix == ".json":
+            with open(path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            for item in data:
+                result = ScrapeResult(**item)
+                results.append(result)
+                urls.add(result.input_url)
+        elif path.suffix == ".csv":
+            with open(path, "r", encoding="utf-8") as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    urls.add(row.get("input_url", ""))
+                    # For CSV resume, we just track URLs, not full reconstruction
+    except Exception as e:
+        logger.warning("Failed to load existing results from %s: %s", path, e)
+
+    return results, urls
+
+
+def _save_results(results: list[ScrapeResult], path: Path) -> None:
+    """Save results to JSON or CSV file."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if path.suffix == ".csv":
+        _save_csv(results, path)
+    else:
+        _save_json(results, path)
+
+
+def _save_json(results: list[ScrapeResult], path: Path) -> None:
+    """Save results as JSON."""
+    data = []
+    for r in results:
+        item = r.model_dump(mode="json")
+        data.append(item)
+
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2, default=str)
+
+
+def _save_csv(results: list[ScrapeResult], path: Path) -> None:
+    """Save results as flat CSV (place details only, no reviews)."""
+    if not results:
+        return
+
+    fieldnames = [
+        "input_url",
+        "success",
+        "error",
+        "scraped_at",
+        "name",
+        "place_id",
+        "address",
+        "rating",
+        "review_count",
+        "phone",
+        "website",
+        "category",
+        "price_level",
+        "latitude",
+        "longitude",
+        "plus_code",
+        "url",
+        "google_maps_url",
+        "description",
+        "photos_count",
+        "permanently_closed",
+        "temporarily_closed",
+    ]
+
+    with open(path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
+        writer.writeheader()
+
+        for r in results:
+            row = {
+                "input_url": r.input_url,
+                "success": r.success,
+                "error": r.error,
+                "scraped_at": str(r.scraped_at),
+            }
+            if r.place:
+                place_dict = r.place.model_dump()
+                # Flatten hours to a single string
+                if place_dict.get("hours"):
+                    place_dict["hours"] = " | ".join(place_dict["hours"])
+                row.update(place_dict)
+            writer.writerow(row)
+
+
+def load_urls_from_file(path: str | Path) -> list[str]:
+    """Load URLs from a text file or CSV file.
+
+    Supports:
+    - Plain text file (one URL per line)
+    - CSV file (looks for 'url' column, or uses first column)
+
+    Args:
+        path: Path to the input file.
+
+    Returns:
+        List of URL strings.
+    """
+    path = Path(path)
+
+    if path.suffix == ".csv":
+        urls = []
+        with open(path, "r", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            # Try 'url' column first, then first column
+            for row in reader:
+                url = row.get("url") or row.get("URL") or row.get("link")
+                if not url:
+                    # Use first column value
+                    url = next(iter(row.values()), None)
+                if url and url.strip():
+                    urls.append(url.strip())
+        return urls
+    else:
+        # Plain text, one URL per line
+        with open(path, "r", encoding="utf-8") as f:
+            return [
+                line.strip()
+                for line in f
+                if line.strip() and not line.startswith("#")
+            ]

gmaps_scraper/cli.py

@@ -0,0 +1,238 @@
+"""CLI entry point for gmaps-scraper."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+from pathlib import Path
+
+from gmaps_scraper.batch import load_urls_from_file, scrape_batch
+from gmaps_scraper.models import ScrapeConfig, ScrapeResult
+from gmaps_scraper.scraper import GoogleMapsScraper
+
+
+def main() -> None:
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        prog="gmaps-scraper",
+        description="Scrape Google Maps place details without API key",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="Enable verbose logging",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # --- scrape command ---
+    scrape_parser = subparsers.add_parser(
+        "scrape",
+        help="Scrape a single Google Maps URL",
+    )
+    scrape_parser.add_argument(
+        "url",
+        help="Google Maps URL to scrape",
+    )
+    scrape_parser.add_argument(
+        "--lang",
+        default=None,
+        help="Language code (e.g., zh-TW, en, ja). Default: none",
+    )
+    scrape_parser.add_argument(
+        "--no-headless",
+        action="store_true",
+        help="Show browser window (for debugging)",
+    )
+    scrape_parser.add_argument(
+        "--reviews",
+        action="store_true",
+        help="Also scrape individual reviews",
+    )
+    scrape_parser.add_argument(
+        "--max-reviews",
+        type=int,
+        default=20,
+        help="Max reviews to scrape (default: 20)",
+    )
+
+    # --- batch command ---
+    batch_parser = subparsers.add_parser(
+        "batch",
+        help="Batch scrape URLs from a file",
+    )
+    batch_parser.add_argument(
+        "input",
+        help="Input file (CSV or text, one URL per line)",
+    )
+    batch_parser.add_argument(
+        "-o", "--output",
+        required=True,
+        help="Output file path (.json or .csv)",
+    )
+    batch_parser.add_argument(
+        "--concurrency",
+        type=int,
+        default=5,
+        help="Number of concurrent scrapers (default: 5)",
+    )
+    batch_parser.add_argument(
+        "--lang",
+        default=None,
+        help="Language code (e.g., zh-TW, en, ja). Default: none",
+    )
+    batch_parser.add_argument(
+        "--no-headless",
+        action="store_true",
+        help="Show browser window (for debugging)",
+    )
+    batch_parser.add_argument(
+        "--proxy",
+        help="Proxy server URL (e.g., http://proxy:8080)",
+    )
+    batch_parser.add_argument(
+        "--delay-min",
+        type=float,
+        default=2.0,
+        help="Minimum delay between requests in seconds (default: 2.0)",
+    )
+    batch_parser.add_argument(
+        "--delay-max",
+        type=float,
+        default=5.0,
+        help="Maximum delay between requests in seconds (default: 5.0)",
+    )
+    batch_parser.add_argument(
+        "--no-resume",
+        action="store_true",
+        help="Don't resume from existing output file",
+    )
+    batch_parser.add_argument(
+        "--reviews",
+        action="store_true",
+        help="Also scrape individual reviews",
+    )
+    batch_parser.add_argument(
+        "--max-reviews",
+        type=int,
+        default=20,
+        help="Max reviews per place (default: 20)",
+    )
+    batch_parser.add_argument(
+        "--save-interval",
+        type=int,
+        default=50,
+        help="Save results every N items (default: 50)",
+    )
+
+    args = parser.parse_args()
+
+    # Setup logging
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    if args.command == "scrape":
+        _cmd_scrape(args)
+    elif args.command == "batch":
+        _cmd_batch(args)
+
+
+def _cmd_scrape(args: argparse.Namespace) -> None:
+    """Handle `gmaps-scraper scrape` command."""
+    config = ScrapeConfig(
+        headless=not args.no_headless,
+        language=args.lang,
+        scrape_reviews=args.reviews,
+        max_reviews=args.max_reviews,
+    )
+
+    async def _run() -> ScrapeResult:
+        async with GoogleMapsScraper(config) as scraper:
+            return await scraper.scrape(args.url)
+
+    result = asyncio.run(_run())
+
+    # Output as JSON
+    output = result.model_dump(mode="json")
+    print(json.dumps(output, ensure_ascii=False, indent=2, default=str))
+
+    if not result.success:
+        sys.exit(1)
+
+
+def _cmd_batch(args: argparse.Namespace) -> None:
+    """Handle `gmaps-scraper batch` command."""
+    input_path = Path(args.input)
+    if not input_path.exists():
+        print(f"Error: Input file not found: {input_path}", file=sys.stderr)
+        sys.exit(1)
+
+    urls = load_urls_from_file(input_path)
+    if not urls:
+        print("Error: No URLs found in input file", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Loaded {len(urls)} URLs from {input_path}")
+
+    config = ScrapeConfig(
+        concurrency=args.concurrency,
+        headless=not args.no_headless,
+        language=args.lang,
+        proxy=args.proxy,
+        delay_min=args.delay_min,
+        delay_max=args.delay_max,
+        save_interval=args.save_interval,
+        scrape_reviews=args.reviews,
+        max_reviews=args.max_reviews,
+    )
+
+    # Progress tracking
+    try:
+        from tqdm import tqdm
+
+        pbar = tqdm(total=len(urls), desc="Scraping", unit="place")
+
+        def on_result(result: ScrapeResult, idx: int, total: int) -> None:
+            pbar.update(1)
+            status = "✓" if result.success else "✗"
+            name = result.place.name if result.place else "?"
+            pbar.set_postfix_str(f"{status} {name}")
+
+    except ImportError:
+
+        def on_result(result: ScrapeResult, idx: int, total: int) -> None:
+            status = "✓" if result.success else "✗"
+            name = result.place.name if result.place else "?"
+            print(f"  [{idx}/{total}] {status} {name}")
+
+        pbar = None
+
+    results = asyncio.run(
+        scrape_batch(
+            urls=urls,
+            config=config,
+            output_path=args.output,
+            on_result=on_result,
+            resume=not args.no_resume,
+        )
+    )
+
+    if pbar:
+        pbar.close()
+
+    # Summary
+    success_count = sum(1 for r in results if r.success)
+    fail_count = len(results) - success_count
+    print(f"\nDone! {success_count} succeeded, {fail_count} failed")
+    print(f"Results saved to: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

gmaps_scraper/models.py

@@ -0,0 +1,82 @@
+"""Pydantic data models for Google Maps scraper."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class PlaceDetails(BaseModel):
+    """Structured details about a Google Maps place."""
+
+    name: Optional[str] = None
+    place_id: Optional[str] = None
+    address: Optional[str] = None
+    rating: Optional[float] = None
+    review_count: Optional[int] = None
+    phone: Optional[str] = None
+    website: Optional[str] = None
+    category: Optional[str] = None
+    price_level: Optional[str] = None
+    hours: Optional[list[str]] = None
+    latitude: Optional[float] = None
+    longitude: Optional[float] = None
+    plus_code: Optional[str] = None
+    url: Optional[str] = None
+    google_maps_url: Optional[str] = None
+    image_url: Optional[str] = None
+    description: Optional[str] = None
+    photos_count: Optional[int] = None
+
+    # Additional details
+    permanently_closed: bool = False
+    temporarily_closed: bool = False
+
+
+class Review(BaseModel):
+    """A single Google Maps review."""
+
+    author: Optional[str] = None
+    rating: Optional[int] = None
+    text: Optional[str] = None
+    time: Optional[str] = None
+    language: Optional[str] = None
+    likes: Optional[int] = None
+
+
+class ScrapeResult(BaseModel):
+    """Result of scraping a single Google Maps URL."""
+
+    input_url: str
+    place: Optional[PlaceDetails] = None
+    reviews: list[Review] = Field(default_factory=list)
+    success: bool = True
+    error: Optional[str] = None
+    scraped_at: datetime = Field(default_factory=datetime.now)
+
+
+class ScrapeConfig(BaseModel):
+    """Configuration for the scraper."""
+
+    concurrency: int = Field(default=5, ge=1, le=50)
+    timeout: int = Field(default=30000, description="Page load timeout in ms")
+    headless: bool = True
+    language: Optional[str] = None
+    max_retries: int = 3
+    delay_min: float = 1.0
+    delay_max: float = 3.0
+    save_interval: int = Field(
+        default=50,
+        description="Save results every N items for crash recovery",
+    )
+    proxy: Optional[str] = None
+    scrape_reviews: bool = Field(
+        default=False,
+        description="Whether to also scrape individual reviews",
+    )
+    max_reviews: int = Field(
+        default=20,
+        description="Max reviews to scrape per place (if scrape_reviews=True)",
+    )