polite-scraper 1.0.0__py3-none-any.whl

polite_scraper/__init__.py

@@ -0,0 +1,3 @@
+"""The Polite Scraper — a rate-limited, cache-first web scraper."""
+
+__version__ = "1.0.0"

polite_scraper/__main__.py

@@ -0,0 +1,4 @@
+from polite_scraper.main import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

polite_scraper/cli.py

@@ -0,0 +1,186 @@
+"""Command-line interface for The Polite Scraper."""
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+from polite_scraper.config import Config
+
+USAGE_EXAMPLES = """\
+[bold]Examples:[/]
+  python -m polite_scraper https://books.toscrape.com
+  python -m polite_scraper https://en.wikipedia.org --mode generic --pages 10
+  python -m polite_scraper https://books.toscrape.com --mode book --pages 5 --force
+  python -m polite_scraper https://example.com -v          # verbose logs
+"""
+
+
+class Parser(argparse.ArgumentParser):
+    def error(self, message: str):  # pragma: no cover - argparse hook
+        from polite_scraper import ui
+
+        ui.error(f"{message}\n")
+        ui.console.print(USAGE_EXAMPLES)
+        raise SystemExit(2)
+
+
+def build_parser() -> Parser:
+    parser = Parser(
+        prog="polite-scraper",
+        description="A polite, rate-limited web scraper with caching and validation.",
+        epilog=USAGE_EXAMPLES,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "url",
+        nargs="?",
+        default=None,
+        help="Website to scrape (e.g. https://books.toscrape.com)",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["auto", "book", "generic"],
+        default="auto",
+        help=(
+            "Scraper mode: 'book' uses the catalogue/product-page extractor, "
+            "'generic' extracts page metadata from any site, "
+            "'auto' detects book catalogues and falls back to generic (default: auto)"
+        ),
+    )
+    parser.add_argument(
+        "--pages",
+        type=int,
+        default=3,
+        help="Maximum number of pages to crawl (default: 3)",
+    )
+    parser.add_argument(
+        "--max-links",
+        type=int,
+        default=8,
+        help="Generic mode only: max new same-domain links followed per page (default: 8)",
+    )
+    parser.add_argument(
+        "--delay",
+        type=float,
+        default=None,
+        help="Minimum delay in seconds between network requests (default: 0.5)",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=None,
+        help="Network request timeout in seconds (default: 10)",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Ignore the local HTML cache and re-fetch everything",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Show detailed per-request logs and raw record checkpoints",
+    )
+    parser.add_argument(
+        "--name",
+        default=None,
+        help="Custom name for this run's output folder (default: derived from the website name)",
+    )
+    return parser
+
+
+def build_config(args: argparse.Namespace) -> Config:
+    """Map parsed CLI arguments onto a Config instance."""
+    config = Config()
+
+    if args.url:
+        url = args.url.strip()
+        if not url.startswith(("http://", "https://")):
+            url = f"https://{url}"
+        config.BASE_URL = url if url.endswith("/") else f"{url}/"
+
+    if args.delay is not None:
+        config.MIN_DELAY_SECONDS = args.delay
+    if args.timeout is not None:
+        config.TIMEOUT = args.timeout
+    config.MAX_CATALOGUE_PAGES = max(1, args.pages)
+    config.FORCE_REFRESH = args.force
+    config.MAX_LINKS_PER_PAGE = max(1, args.max_links)
+
+    # Environment variables still work, but CLI flags win.
+    env_ua = os.getenv("USER_AGENT")
+    if env_ua:
+        config.USER_AGENT = env_ua
+
+    return config
+
+
+def derive_run_name(base_url: str) -> str:
+    """Derive a filesystem-safe run folder name from the website, e.g.
+    'https://www.books.toscrape.com/' -> 'books-toscrape'."""
+    host = urlparse(base_url).netloc.lower()
+    host = re.sub(r"^www\.", "", host)
+    name = re.sub(r"[^a-z0-9-]+", "-", host).strip("-")
+    return name or "run"
+
+
+def resolve_output_dir(config: Config, explicit_name: str | None) -> Path:
+    """
+    Resolve the per-run output folder under config.OUTPUT_ROOT.
+
+    The default name comes from the website (e.g. 'books-toscrape'). If the
+    folder already exists, the user is asked to pick a new name or accept an
+    auto-suffix ('books-toscrape-1', 'books-toscrape-2', ...). In
+    non-interactive sessions the auto-suffix is applied silently, so runs
+    never overwrite a previous project's output.
+    """
+    base_name = (explicit_name or derive_run_name(config.BASE_URL)).strip()
+    base_name = re.sub(r"[^a-zA-Z0-9_-]+", "-", base_name).strip("-") or "run"
+
+    candidate = config.OUTPUT_ROOT / base_name
+    if not candidate.exists():
+        return candidate
+
+    # Collision: find the next free suffix
+    suffix = 1
+    while (config.OUTPUT_ROOT / f"{base_name}-{suffix}").exists():
+        suffix += 1
+    auto_name = f"{base_name}-{suffix}"
+
+    if not sys.stdin.isatty():
+        from polite_scraper import ui
+
+        ui.info(f"Output folder '{base_name}' already exists — using '{auto_name}'")
+        return config.OUTPUT_ROOT / auto_name
+
+    # Interactive: let the user rename or accept the auto-suffix
+    from polite_scraper import ui
+
+    ui.warn(f"Output folder '{base_name}' already exists.")
+    while True:
+        answer = input(
+            f"  New folder name [{auto_name}]: "
+        ).strip()
+        answer = re.sub(r"[^a-zA-Z0-9_-]+", "-", answer).strip("-")
+        if not answer:
+            return config.OUTPUT_ROOT / auto_name
+        if not (config.OUTPUT_ROOT / answer).exists():
+            return config.OUTPUT_ROOT / answer
+        ui.warn(f"'{answer}' also exists — pick another name")
+
+
+def ensure_url_provided(args: argparse.Namespace) -> bool:
+    from polite_scraper import ui
+
+    if args.url:
+        return True
+    ui.console.print(
+        "[bold red]✗[/] No website given.\n"
+        "  [dim]Usage:[/] polite-scraper [bold]<url>[/] [dim][options][/] \n"
+    )
+    ui.console.print(USAGE_EXAMPLES)
+    return False

polite_scraper/config.py

@@ -0,0 +1,40 @@
+import os
+from pathlib import Path
+from dataclasses import dataclass, field
+
+@dataclass
+class Config:
+    USER_AGENT: str = os.getenv(
+        "USER_AGENT",
+        "PoliteScraper/1.0 (+https://github.com/callmekriztal/polite_scapper)"
+    )
+    TIMEOUT: int = int(os.getenv("TIMEOUT", "10"))
+    MIN_DELAY_SECONDS: float = float(os.getenv("MIN_DELAY_SECONDS", "0.5"))
+    
+    # Data paths are relative to the working directory, not the package location,
+    # so an installed copy (site-packages) never writes inside itself.
+    # From the repo root this resolves to the same <repo>/data layout as before.
+    CACHE_DIR: Path = field(default_factory=lambda: Path.cwd() / "data" / "cache")
+    # Root for per-run output folders (e.g. data/runs/books-toscrape)
+    OUTPUT_ROOT: Path = field(default_factory=lambda: Path.cwd() / "data" / "runs")
+    # The active run's output folder, resolved at runtime from the website name
+    OUTPUT_DIR: Path = field(default_factory=lambda: Path.cwd() / "data" / "runs")
+    
+    BASE_URL: str = "https://books.toscrape.com/"
+    START_URL: str = "https://books.toscrape.com/catalogue/page-1.html"
+    MAX_CATALOGUE_PAGES: int = 3
+
+    # Generic-mode settings (set by the CLI)
+    FORCE_REFRESH: bool = False
+    MAX_LINKS_PER_PAGE: int = 8
+
+    # Simulation hook for broken detail page testing
+    BROKEN_TEST_URL: str | None = os.getenv("BROKEN_TEST_URL", None)
+
+    def toscrape_start_url(self) -> str:
+        """The Books to Scrape catalogue root, used when the user passes the site root."""
+        return self.BASE_URL.rstrip("/") + "/catalogue/page-1.html"
+
+    def ensure_directories(self) -> None:
+        self.CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        self.OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

polite_scraper/crawler.py

@@ -0,0 +1,59 @@
+from bs4 import BeautifulSoup
+from polite_scraper.config import Config
+from polite_scraper.fetcher import Fetcher
+from polite_scraper.utils import make_absolute_url
+
+class DiscoveryResult:
+    def __init__(self, catalogue_pages_count: int, discovered_count: int, book_urls: list[dict]):
+        self.catalogue_pages_count = catalogue_pages_count
+        self.discovered_count = discovered_count
+        self.book_urls = book_urls  # List of dicts: {"product_url": ..., "source_page": ...}
+
+class CatalogueCrawler:
+    """Discovers book URLs by crawling catalogue pagination."""
+
+    def __init__(self, config: Config, fetcher: Fetcher):
+        self.config = config
+        self.fetcher = fetcher
+
+    def discover_books(self) -> DiscoveryResult:
+        current_url = self.config.START_URL
+        pages_processed = 0
+        discovered_entries = []
+        seen_urls = set()
+
+        while current_url and pages_processed < self.config.MAX_CATALOGUE_PAGES:
+            html_content, _ = self.fetcher.fetch(current_url)
+            pages_processed += 1
+
+            soup = BeautifulSoup(html_content, "html.parser")
+            
+            # Find all product links in the current catalogue page
+            # Each book card is in an <article class="product_pod">
+            product_pods = soup.select("article.product_pod")
+            for pod in product_pods:
+                a_tag = pod.select_one("h3 a")
+                if a_tag and a_tag.get("href"):
+                    rel_url = a_tag["href"]
+                    abs_url = make_absolute_url(current_url, rel_url)
+                    
+                    if abs_url not in seen_urls:
+                        seen_urls.add(abs_url)
+                        discovered_entries.append({
+                            "product_url": abs_url,
+                            "source_page": current_url
+                        })
+
+            # Check for "next" page link
+            next_li = soup.select_one("li.next a")
+            if next_li and next_li.get("href"):
+                next_rel = next_li["href"]
+                current_url = make_absolute_url(current_url, next_rel)
+            else:
+                current_url = None
+
+        return DiscoveryResult(
+            catalogue_pages_count=pages_processed,
+            discovered_count=len(discovered_entries),
+            book_urls=discovered_entries
+        )

polite_scraper/extractor.py

@@ -0,0 +1,92 @@
+from datetime import datetime, timezone
+import requests
+from bs4 import BeautifulSoup
+from polite_scraper.config import Config
+from polite_scraper.fetcher import Fetcher
+from polite_scraper.utils import (
+    clean_text,
+    normalize_availability,
+    normalize_price,
+    normalize_rating
+)
+
+class BookExtractor:
+    """Extracts raw and normalized product data from book detail pages."""
+
+    def __init__(self, config: Config, fetcher: Fetcher):
+        self.config = config
+        self.fetcher = fetcher
+
+    def extract_book(self, product_url: str, source_page: str) -> dict:
+        """
+        Fetch book detail page and extract fields.
+        Raises Exception if page retrieval or parsing fails.
+        """
+        # Simulation hook for broken test URL
+        if self.config.BROKEN_TEST_URL and product_url == self.config.BROKEN_TEST_URL:
+            raise requests.HTTPError(f"Simulated broken page error for {product_url}")
+
+        fetched_at = datetime.now(timezone.utc).isoformat()
+        html_content, _ = self.fetcher.fetch(product_url)
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        # Specific selector for product main area
+        product_main = soup.select_one("div.product_main")
+        if not product_main:
+            raise ValueError(f"Could not find product_main area in {product_url}")
+
+        # Title
+        h1_tag = product_main.select_one("h1")
+        title_raw = h1_tag.get_text() if h1_tag else ""
+        title = clean_text(title_raw) or ""
+
+        # Price text
+        price_tag = product_main.select_one("p.price_color")
+        price_text = clean_text(price_tag.get_text()) if price_tag else ""
+
+        # Availability text
+        avail_tag = product_main.select_one("p.instock.availability")
+        availability_text = clean_text(avail_tag.get_text()) if avail_tag else ""
+
+        # Rating text
+        rating_tag = product_main.select_one("p.star-rating")
+        rating_text = ""
+        if rating_tag:
+            classes = rating_tag.get("class", [])
+            for c in classes:
+                if c.lower() != "star-rating":
+                    rating_text = c
+                    break
+
+        # Description (located after #product_description div header if present)
+        desc_header = soup.select_one("#product_description")
+        description = None
+        if desc_header:
+            desc_p = desc_header.find_next_sibling("p")
+            if desc_p:
+                description = clean_text(desc_p.get_text())
+
+        # Construct raw record
+        raw_record = {
+            "title": title,
+            "product_url": product_url,
+            "price_text": price_text,
+            "availability_text": availability_text,
+            "rating_text": rating_text,
+            "description": description,
+            "source_page": source_page,
+            "fetched_at": fetched_at,
+        }
+
+        # Normalize fields (Stage 4)
+        price_gbp = normalize_price(price_text)
+        rating_int = normalize_rating(rating_text)
+        availability_norm = normalize_availability(availability_text)
+
+        raw_record.update({
+            "price_gbp": price_gbp,
+            "rating_int": rating_int,
+            "availability_normalized": availability_norm
+        })
+
+        return raw_record

polite_scraper/fetcher.py

@@ -0,0 +1,94 @@
+import time
+from urllib.robotparser import RobotFileParser
+
+import requests
+
+from polite_scraper import ui
+from polite_scraper.config import Config
+from polite_scraper.utils import url_to_cache_filename
+
+
+class Fetcher:
+    """HTTP fetcher with polite rate-limiting, status checking, and HTML disk caching."""
+
+    def __init__(self, config: Config):
+        self.config = config
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": self.config.USER_AGENT
+        })
+        self.cache_hits = 0
+        self.network_fetches = 0
+        self.last_request_time = 0.0
+
+    def fetch(self, url: str, force_refresh: bool = False) -> tuple[str, bool]:
+        """
+        Fetch HTML from URL or cache.
+        Returns tuple of (html_content, was_cached).
+        Raises requests.HTTPError if response status != 200.
+        """
+        force_refresh = force_refresh or self.config.FORCE_REFRESH
+        self.config.ensure_directories()
+        cache_filename = url_to_cache_filename(url)
+        cache_path = self.config.CACHE_DIR / cache_filename
+
+        # Check local cache first unless force_refresh is True
+        if not force_refresh and cache_path.is_file():
+            try:
+                html_content = cache_path.read_text(encoding="utf-8")
+                self.cache_hits += 1
+                size_bytes = len(html_content.encode("utf-8"))
+                ui.debug(f"CACHE HIT  {url} ({size_bytes} bytes)")
+                return html_content, True
+            except Exception as e:
+                ui.debug(f"Cache read error for {cache_path}: {e}. Falling back to network fetch.")
+
+        html_content = self._network_get(url)
+        cache_path.write_text(html_content, encoding="utf-8")
+        return html_content, False
+
+    def fetch_robots_txt(self, base_url: str) -> RobotFileParser | None:
+        """
+        Fetch robots.txt for a site. Returns a parser, or None if no robots.txt
+        exists / it cannot be fetched (treated as "no rules").
+        """
+        robots_url = base_url.rstrip("/") + "/robots.txt"
+        try:
+            response = self.session.get(robots_url, timeout=self.config.TIMEOUT)
+        except requests.RequestException as e:
+            ui.debug(f"robots.txt fetch failed for {robots_url}: {e}")
+            return None
+        if response.status_code != 200:
+            ui.debug(f"robots.txt not available ({response.status_code}) at {robots_url}")
+            return None
+
+        parser = RobotFileParser()
+        parser.parse(response.text.splitlines())
+        return parser
+
+    def _network_get(self, url: str) -> str:
+        # Respect rate limit of at least MIN_DELAY_SECONDS between network requests
+        now = time.time()
+        elapsed = now - self.last_request_time
+        if elapsed < self.config.MIN_DELAY_SECONDS:
+            time_to_sleep = self.config.MIN_DELAY_SECONDS - elapsed
+            time.sleep(time_to_sleep)
+
+        # Make network request
+        ui.debug(f"FETCH      {url} ...")
+        self.last_request_time = time.time()
+        response = self.session.get(url, timeout=self.config.TIMEOUT)
+        response.encoding = "utf-8"  # Ensure proper UTF-8 decoding for currency symbols
+        self.network_fetches += 1
+
+        # Only treat HTTP 200 as successful per Stage 1 requirement
+        if response.status_code != 200:
+            raise requests.HTTPError(
+                f"HTTP fetch failed for {url} with status code {response.status_code}",
+                response=response
+            )
+
+        html_content = response.text
+        size_bytes = len(html_content.encode("utf-8"))
+        ui.debug(f"FETCH OK   {url} ({size_bytes} bytes)")
+        return html_content

polite_scraper/generic.py

@@ -0,0 +1,153 @@
+"""Generic site scraper.
+
+Works on any website: BFS-crawls same-domain links (rate-limited and
+robots.txt-checked) and extracts useful page metadata from each page.
+"""
+
+from collections import deque
+from datetime import datetime, timezone
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+from polite_scraper import ui
+from polite_scraper.config import Config
+from polite_scraper.fetcher import Fetcher
+from polite_scraper.utils import clean_text, make_absolute_url
+
+
+class GenericExtractor:
+    """Extracts page metadata from arbitrary HTML."""
+
+    HEADING_TAGS = ["h1", "h2", "h3", "h4", "h5", "h6"]
+
+    def extract_page(self, page_url: str, html_content: str) -> dict:
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        # Title: <title> falls back to first <h1>
+        title = clean_text(soup.title.get_text()) if soup.title else None
+        if not title:
+            h1 = soup.find("h1")
+            title = clean_text(h1.get_text()) if h1 else ""
+
+        # First H1
+        h1 = soup.find("h1")
+        h1_text = clean_text(h1.get_text()) if h1 else None
+
+        # Meta description
+        meta_desc = None
+        meta_tag = soup.find("meta", attrs={"name": "description"})
+        if meta_tag and meta_tag.get("content"):
+            meta_desc = clean_text(meta_tag["content"])
+
+        # Headings (up to 10, non-empty)
+        headings = []
+        for tag in soup.find_all(self.HEADING_TAGS):
+            text = clean_text(tag.get_text())
+            if text:
+                headings.append(text)
+            if len(headings) >= 10:
+                break
+
+        # Same-domain links
+        origin = urlparse(page_url)
+        same_links = []
+        for a in soup.find_all("a", href=True):
+            abs_url = make_absolute_url(page_url, a["href"])
+            parsed = urlparse(abs_url)
+            if parsed.netloc == origin.netloc and parsed.scheme in ("http", "https"):
+                same_links.append(abs_url.split("#")[0])
+        unique_links = list(dict.fromkeys(same_links))
+
+        # Word count of visible-ish text
+        for junk in soup(["script", "style", "noscript"]):
+            junk.decompose()
+        body_text = soup.get_text(separator=" ", strip=True)
+        word_count = len(body_text.split())
+
+        return {
+            "url": page_url,
+            "title": title or "",
+            "h1": h1_text,
+            "meta_description": meta_desc,
+            "headings": headings,
+            "same_domain_links": unique_links[:50],
+            "link_count": len(unique_links),
+            "image_count": len(soup.find_all("img")),
+            "word_count": word_count,
+            "fetched_at": datetime.now(timezone.utc).isoformat(),
+        }
+
+
+class GenericSiteScraper:
+    """BFS crawler + extractor for arbitrary websites."""
+
+    def __init__(self, config: Config, fetcher: Fetcher, extractor: GenericExtractor | None = None):
+        self.config = config
+        self.fetcher = fetcher
+        self.extractor = extractor or GenericExtractor()
+        self.robots_parser = None
+
+    def check_robots(self, start_url: str) -> bool:
+        """Politeness check: consult robots.txt for our User-Agent. Returns True if allowed."""
+        base = f"{urlparse(start_url).scheme}://{urlparse(start_url).netloc}"
+        self.robots_parser = self.fetcher.fetch_robots_txt(base)
+        if self.robots_parser is None:
+            ui.info("robots.txt: not available (no rules) — proceeding politely anyway")
+            return True
+        allowed = self.robots_parser.can_fetch(self.config.USER_AGENT, start_url)
+        if allowed:
+            ui.success("robots.txt: crawl allowed for our User-Agent")
+        else:
+            ui.error("robots.txt disallows scraping this URL for our User-Agent — aborting")
+        return allowed
+
+    def scrape(self) -> tuple[list[dict], list[dict]]:
+        """Returns (raw_page_records, error_records)."""
+        start_url = self.config.START_URL
+        max_pages = self.config.MAX_CATALOGUE_PAGES
+        max_links = self.config.MAX_LINKS_PER_PAGE
+
+        raw_records: list[dict] = []
+        error_records: list[dict] = []
+
+        queue: deque[str] = deque([start_url])
+        seen: set[str] = {start_url.split("#")[0]}
+        pages_fetched = 0
+
+        while queue and pages_fetched < max_pages:
+            url = queue.popleft()
+
+            try:
+                html_content, _ = self.fetcher.fetch(url)
+            except Exception as ex:
+                error_records.append({"url": url, "reason": f"Fetch failed: {ex}"})
+                continue
+
+            pages_fetched += 1
+            raw = self.extractor.extract_page(url, html_content)
+
+            if not raw["title"]:
+                error_records.append({
+                    "url": url,
+                    "reason": "No page title found (skipped record)",
+                    "raw_record": raw,
+                })
+            else:
+                raw_records.append(raw)
+
+            # Enqueue new same-domain links (up to max_links per page)
+            for link in raw["same_domain_links"]:
+                if pages_fetched + len(queue) >= max_pages and len(queue) >= max_links:
+                    break
+                if link not in seen:
+                    seen.add(link)
+                    if self._robots_allows(link):
+                        queue.append(link)
+
+        return raw_records, error_records
+
+    def _robots_allows(self, url: str) -> bool:
+        if self.robots_parser is None:
+            return True
+        return self.robots_parser.can_fetch(self.config.USER_AGENT, url)