scrapper-tool 0.1.0__py3-none-any.whl

scrapper_tool/__init__.py

@@ -0,0 +1,66 @@
+"""scrapper-tool — reusable web-scraping toolkit.
+
+Pattern A/B/C/D ladder, TLS-impersonation fallback chain, deterministic
+fixture-replay testing, and (v0.2.0+) an optional MCP server for LLM agents.
+
+MCP integration is available via ``pip install scrapper-tool[agent]`` —
+see ``docs/agent-integration.md`` (M13).
+
+Public API
+----------
+
+The top-level ``scrapper_tool`` namespace re-exports the most commonly used
+symbols. Submodules expose the rest:
+
+- ``scrapper_tool.http`` — :func:`vendor_client`, :func:`request_with_retry`
+- ``scrapper_tool.errors`` — :class:`VendorHTTPError`, :class:`BlockedError`,
+  :class:`ParseError`, :class:`VendorUnavailable`, :class:`ScrapingError`
+- ``scrapper_tool.ladder`` *(M2)* — ``IMPERSONATE_LADDER`` and the walk logic
+- ``scrapper_tool.patterns.{a,b,c,d}`` *(M3-M5)* — extraction helpers per pattern
+- ``scrapper_tool.testing`` *(M6)* — fixture-replay test helpers
+- ``scrapper_tool.canary`` *(M8)* — CLI for fingerprint-health probes
+- ``scrapper_tool.adapter`` *(M7)* — generic ``Adapter[QueryT, ResultT]`` Protocol
+
+Stability
+---------
+
+This is alpha software (v0.x). The public API may change between minor
+versions until v1.0.0; pin a tilde or caret range when consuming.
+"""
+
+from __future__ import annotations
+
+from scrapper_tool.adapter import Adapter
+from scrapper_tool.errors import (
+    BlockedError,
+    ParseError,
+    ScrapingError,
+    VendorHTTPError,
+    VendorUnavailable,
+)
+from scrapper_tool.http import (
+    VendorHTTPClient,
+    request_with_retry,
+    vendor_client,
+)
+from scrapper_tool.ladder import (
+    IMPERSONATE_LADDER,
+    request_with_ladder,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "IMPERSONATE_LADDER",
+    "Adapter",
+    "BlockedError",
+    "ParseError",
+    "ScrapingError",
+    "VendorHTTPClient",
+    "VendorHTTPError",
+    "VendorUnavailable",
+    "__version__",
+    "request_with_ladder",
+    "request_with_retry",
+    "vendor_client",
+]

scrapper_tool/_logging.py

@@ -0,0 +1,78 @@
+"""Optional structlog binding shim.
+
+The lib should be usable in projects that don't depend on ``structlog``.
+This shim returns a real ``structlog`` logger if available, or a thin
+adapter around stdlib ``logging`` that accepts the same keyword-arg
+shape (``logger.warning("event", key=value)``) so call sites don't
+have to branch.
+
+Usage::
+
+    from scrapper_tool._logging import get_logger
+
+    _logger = get_logger(__name__)
+    _logger.warning("vendor_http.transport_error", url=url, attempt=attempt)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Protocol
+
+
+class _Logger(Protocol):
+    """Minimal structured-logger surface used by the lib."""
+
+    def debug(self, event: str, **kwargs: Any) -> None: ...
+    def info(self, event: str, **kwargs: Any) -> None: ...
+    def warning(self, event: str, **kwargs: Any) -> None: ...
+    def error(self, event: str, **kwargs: Any) -> None: ...
+
+
+class _StdlibStructAdapter:
+    """Wraps a stdlib ``logging.Logger`` to accept ``key=value`` kwargs.
+
+    Renders kwargs as ``key=value key2=value2`` after the event name —
+    not as JSON; the goal is "readable in a plain console" rather than
+    "structured log shipping". Consumers that want JSON should install
+    ``structlog`` themselves; this lib will pick it up automatically.
+    """
+
+    def __init__(self, name: str) -> None:
+        self._inner = logging.getLogger(name)
+
+    @staticmethod
+    def _render(event: str, kwargs: dict[str, Any]) -> str:
+        if not kwargs:
+            return event
+        rendered = " ".join(f"{k}={v!r}" for k, v in kwargs.items())
+        return f"{event} {rendered}"
+
+    def debug(self, event: str, **kwargs: Any) -> None:
+        self._inner.debug(self._render(event, kwargs))
+
+    def info(self, event: str, **kwargs: Any) -> None:
+        self._inner.info(self._render(event, kwargs))
+
+    def warning(self, event: str, **kwargs: Any) -> None:
+        self._inner.warning(self._render(event, kwargs))
+
+    def error(self, event: str, **kwargs: Any) -> None:
+        self._inner.error(self._render(event, kwargs))
+
+
+def get_logger(name: str) -> _Logger:
+    """Return a structured logger for ``name``.
+
+    Prefers ``structlog`` when installed; falls back to a stdlib
+    adapter otherwise. Both implementations satisfy the ``_Logger``
+    protocol so call sites are uniform.
+    """
+    try:
+        import structlog  # noqa: PLC0415
+    except ImportError:
+        return _StdlibStructAdapter(name)
+    return structlog.get_logger(name)  # type: ignore[no-any-return,unused-ignore]
+
+
+__all__ = ["get_logger"]

scrapper_tool/adapter.py

@@ -0,0 +1,127 @@
+"""Generic ``Adapter[QueryT, ResultT]`` Protocol.
+
+A specialisation contract for vendor-specific scrapers built on top of
+``scrapper-tool``. Consumers implement a class that conforms to this
+Protocol and gain typed integration with the rest of the lib (notably
+the M13 MCP server).
+
+Why a Protocol (not an ABC)
+---------------------------
+
+`scrapper-tool` doesn't impose inheritance — adapters can be plain
+classes, dataclasses, attrs models, or whatever the consumer prefers.
+The Protocol is structural: any object exposing ``vendor_id``,
+``search()``, and ``fetch_detail()`` with the right shapes satisfies
+``Adapter[Q, R]``. ``isinstance(x, Adapter)`` works because the
+Protocol is ``runtime_checkable``.
+
+Specialising
+------------
+
+::
+
+    from dataclasses import dataclass
+    from decimal import Decimal
+    from pydantic import BaseModel
+
+    from scrapper_tool import vendor_client, request_with_retry
+    from scrapper_tool.patterns.b import extract_product_offer
+    from scrapper_tool.adapter import Adapter
+
+    class OEMQuery(BaseModel):
+        oem: str
+
+    class PartResult(BaseModel):
+        oem: str
+        vendor: str
+        price: Decimal
+        currency: str
+        url: str
+
+    @dataclass
+    class MyVendorAdapter:  # implements Adapter[OEMQuery, PartResult]
+        vendor_id: str = "myvendor"
+
+        async def search(self, query: OEMQuery) -> list[PartResult]:
+            url = f"https://myvendor.example/api/search?oem={query.oem}"
+            async with vendor_client() as client:
+                resp = await request_with_retry(client, "GET", url)
+            ...   # parse + return
+
+        async def fetch_detail(self, url: str) -> PartResult | None:
+            async with vendor_client() as client:
+                resp = await request_with_retry(client, "GET", url)
+            offer = extract_product_offer(resp.text, base_url=url)
+            ...   # convert to PartResult
+
+    adapter: Adapter[OEMQuery, PartResult] = MyVendorAdapter()
+
+The Protocol is ``runtime_checkable``, so ``isinstance(adapter, Adapter)``
+returns True at runtime without inheritance.
+
+Consumers wanting domain-specific extensions (PartsPilot's
+``VendorAdapter`` is one) can subclass the Protocol or define their
+own narrower one. The lib stays out of domain modelling.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Adapter[QueryT, ResultT](Protocol):
+    """Generic vendor-adapter contract.
+
+    Type parameters
+    ---------------
+    QueryT
+        The query / input type. PartsPilot uses an ``OEMQuery``
+        Pydantic model with vehicle context; weather scrapers use a
+        ``Location`` model; etc.
+    ResultT
+        The result type. Should be a Pydantic model so consumers can
+        snapshot-test against goldens (see
+        :func:`scrapper_tool.testing.assert_pydantic_snapshot`).
+
+    Required attributes
+    -------------------
+    vendor_id : str
+        Stable, lowercase, hyphen-or-underscore-free identifier for the
+        vendor (e.g. ``"amayama"``, ``"megazip"``). Used by the lib's
+        logging + the MCP server (M13) to tag tool calls.
+
+    Required methods
+    ----------------
+    ``async search(query: QueryT) -> list[ResultT]``
+        Search the vendor for results matching ``query``. Return a
+        list (possibly empty) of result records.
+    ``async fetch_detail(url: str) -> ResultT | None``
+        Fetch a single result detail from a canonical URL. Return
+        ``None`` when the vendor returns 404 or the URL doesn't
+        resolve to a vendor result.
+
+    Both methods are expected to:
+    - Wrap the actual HTTP work in :func:`scrapper_tool.http.vendor_client`
+      and :func:`scrapper_tool.http.request_with_retry`.
+    - Wrap the call site in the consumer's own circuit breaker — the
+      lib provides primitives but does not own breaker policy.
+    - Raise :class:`scrapper_tool.errors.VendorHTTPError` /
+      :class:`scrapper_tool.errors.VendorUnavailable` on transport
+      exhaustion (covers the breaker's "vendor down" branch).
+    - Raise :class:`scrapper_tool.errors.BlockedError` when the
+      anti-bot ladder exhausts (consumer should escalate to Pattern D).
+    - Raise :class:`scrapper_tool.errors.ParseError` on parser drift
+      — DOES NOT trip the breaker (it's "our bug", not "their fault").
+    """
+
+    vendor_id: str
+
+    async def search(self, query: QueryT) -> list[ResultT]: ...
+
+    async def fetch_detail(self, url: str) -> ResultT | None: ...
+
+
+__all__ = [
+    "Adapter",
+]

scrapper_tool/canary.py

@@ -0,0 +1,297 @@
+"""``scrapper-tool canary`` CLI — fingerprint-health probe.
+
+Walks the impersonation ladder against a target URL and reports which
+profiles return 200 vs 403/blocked. Designed to run from cron / GitHub
+Actions to surface "chrome133a is starting to 403" before any consumer
+adapter notices.
+
+Usage::
+
+    scrapper-tool canary https://example.com/api/health
+    scrapper-tool canary https://example.com/api/health --json
+    scrapper-tool canary https://example.com/api/health \\
+        --profiles chrome133a,chrome124,safari18_0
+
+Exit codes
+----------
+
+- ``0`` — at least one profile returned ≠ 403/503.
+- ``1`` — all profiles blocked (informs the caller that Pattern D is
+  needed, the URL is hostile, or the canary URL itself moved).
+- ``2`` — argument parsing / runtime error (network down, bad URL).
+
+Output (default, human-readable)::
+
+    URL: https://example.com/api/health
+    Effective profile: chrome133a
+    Profile  | Status | Time (ms)
+    -------- | ------ | ---------
+    chrome133a | 200  |   234
+    chrome124  | -    |     -    (skipped — earlier profile won)
+    safari18_0 | -    |     -
+    firefox135 | -    |     -
+
+Output (--json, machine-readable)::
+
+    {
+      "url": "https://example.com/api/health",
+      "winning_profile": "chrome133a",
+      "exit_code": 0,
+      "results": [
+        {"profile": "chrome133a", "status": 200, "elapsed_ms": 234, "skipped": false},
+        {"profile": "chrome124", "status": null, "elapsed_ms": null, "skipped": true},
+        ...
+      ]
+    }
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+import time
+from typing import TYPE_CHECKING, cast
+
+from scrapper_tool._logging import get_logger
+from scrapper_tool.errors import BlockedError, VendorHTTPError
+from scrapper_tool.http import request_with_retry
+from scrapper_tool.ladder import (
+    IMPERSONATE_LADDER,
+    _curl_cffi_session,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    import httpx
+
+_logger = get_logger(__name__)
+
+
+_ROTATE_STATUS_CODES = frozenset({403, 503})
+
+
+async def probe_profile(
+    profile: str,
+    url: str,
+    *,
+    timeout: float = 10.0,  # noqa: ASYNC109 — passed to curl_cffi, not asyncio.timeout
+    proxy: str | None = None,
+) -> tuple[int | None, float | None, str | None]:
+    """Issue one GET against ``url`` impersonating ``profile``.
+
+    Returns ``(status_code, elapsed_ms, error_message)``. On transport
+    error, status is ``None`` and ``error_message`` carries the
+    failure reason; otherwise ``error_message`` is ``None``.
+    """
+    started = time.perf_counter()
+    try:
+        async with _curl_cffi_session(
+            profile, timeout=timeout, proxy=proxy, extra_headers=None
+        ) as session:
+            resp = await request_with_retry(
+                cast("httpx.AsyncClient", session),
+                "GET",
+                url,
+            )
+        elapsed_ms = (time.perf_counter() - started) * 1000.0
+    except VendorHTTPError as exc:
+        elapsed_ms = (time.perf_counter() - started) * 1000.0
+        return None, elapsed_ms, str(exc)
+    return int(resp.status_code), elapsed_ms, None
+
+
+async def run_canary(
+    url: str,
+    *,
+    ladder: tuple[str, ...] = IMPERSONATE_LADDER,
+    timeout: float = 10.0,  # noqa: ASYNC109 — passed through to curl_cffi
+    proxy: str | None = None,
+) -> dict[str, object]:
+    """Walk ``ladder`` against ``url``. Stop at the first ≠ 403/503.
+
+    Returns a structured result dict (see module docstring's --json
+    example). Profiles tried *after* the winning one are recorded
+    with ``skipped=True``.
+    """
+    if not ladder:
+        msg = "ladder must contain at least one profile"
+        raise ValueError(msg)
+
+    results: list[dict[str, object]] = []
+    winning_profile: str | None = None
+
+    for i, profile in enumerate(ladder):
+        if winning_profile is not None:
+            results.append(
+                {
+                    "profile": profile,
+                    "status": None,
+                    "elapsed_ms": None,
+                    "skipped": True,
+                    "error": None,
+                }
+            )
+            continue
+
+        status, elapsed_ms, error = await probe_profile(profile, url, timeout=timeout, proxy=proxy)
+        results.append(
+            {
+                "profile": profile,
+                "status": status,
+                "elapsed_ms": (round(elapsed_ms, 1) if elapsed_ms is not None else None),
+                "skipped": False,
+                "error": error,
+            }
+        )
+
+        if status is not None and status not in _ROTATE_STATUS_CODES:
+            winning_profile = profile
+            # Mark remaining profiles as skipped.
+            for skipped_profile in ladder[i + 1 :]:
+                results.append(
+                    {
+                        "profile": skipped_profile,
+                        "status": None,
+                        "elapsed_ms": None,
+                        "skipped": True,
+                        "error": None,
+                    }
+                )
+            break
+
+    return {
+        "url": url,
+        "winning_profile": winning_profile,
+        "exit_code": 0 if winning_profile is not None else 1,
+        "results": results,
+    }
+
+
+def _format_text(report: dict[str, object]) -> str:
+    """Render the canary report as a human-readable text table."""
+    lines = [
+        f"URL: {report['url']}",
+        f"Effective profile: {report['winning_profile'] or '(none — all blocked)'}",
+        "",
+        "Profile     | Status | Time (ms) | Skipped",
+        "----------- | ------ | --------- | -------",
+    ]
+    results = report["results"]
+    assert isinstance(results, list)
+    for row in results:
+        assert isinstance(row, dict)
+        status = row["status"]
+        elapsed = row["elapsed_ms"]
+        skipped = row["skipped"]
+        status_cell = "-" if status is None else str(status)
+        elapsed_cell = "-" if elapsed is None else f"{elapsed:.1f}"
+        skipped_cell = "yes" if skipped else "no"
+        profile_str = str(row["profile"])
+        lines.append(f"{profile_str:<11} | {status_cell:<6} | {elapsed_cell:<9} | {skipped_cell}")
+        error = row.get("error")
+        if error:
+            lines.append(f"            error: {error}")
+    return "\n".join(lines)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="scrapper-tool",
+        description="Reusable web-scraping toolkit — Pattern A/B/C/D ladder.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    canary = sub.add_parser(
+        "canary",
+        help="Probe a URL through the impersonation ladder; report which profile won.",
+        description=(
+            "Walks the four-profile impersonation ladder "
+            "(chrome133a -> chrome124 -> safari18_0 -> firefox135) against "
+            "URL. Stops at the first non-403/503. Exit 0 on success, 1 if "
+            "all profiles 403, 2 on error."
+        ),
+    )
+    canary.add_argument("url", help="Target URL to probe.")
+    canary.add_argument(
+        "--profiles",
+        type=str,
+        default=None,
+        help=(
+            "Comma-separated impersonation profiles to walk in order. "
+            "Default: the lib's IMPERSONATE_LADDER."
+        ),
+    )
+    canary.add_argument(
+        "--timeout",
+        type=float,
+        default=10.0,
+        help="Per-request timeout in seconds. Default: 10.0",
+    )
+    canary.add_argument(
+        "--proxy",
+        type=str,
+        default=None,
+        help="Proxy URL to route requests through. Default: none.",
+    )
+    canary.add_argument(
+        "--json",
+        action="store_true",
+        help="Emit machine-readable JSON instead of a text table.",
+    )
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Entry point for ``scrapper-tool`` console script."""
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command != "canary":  # pragma: no cover — argparse rejects others
+        parser.error(f"unknown command: {args.command}")
+
+    ladder: tuple[str, ...]
+    if args.profiles:
+        ladder = tuple(p.strip() for p in args.profiles.split(",") if p.strip())
+        if not ladder:
+            parser.error("--profiles must contain at least one non-empty entry")
+    else:
+        ladder = IMPERSONATE_LADDER
+
+    try:
+        report = asyncio.run(
+            run_canary(
+                args.url,
+                ladder=ladder,
+                timeout=args.timeout,
+                proxy=args.proxy,
+            )
+        )
+    except (BlockedError, ValueError) as exc:
+        sys.stderr.write(f"canary error: {exc}\n")
+        return 2
+
+    if args.json:
+        sys.stdout.write(json.dumps(report, indent=2))
+        sys.stdout.write("\n")
+    else:
+        sys.stdout.write(_format_text(report))
+        sys.stdout.write("\n")
+
+    exit_code = report["exit_code"]
+    assert isinstance(exit_code, int)
+    return exit_code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())
+
+
+__all__ = [
+    "main",
+    "probe_profile",
+    "run_canary",
+]

scrapper_tool/errors.py

@@ -0,0 +1,68 @@
+"""Exception hierarchy for scrapper-tool.
+
+::
+
+    ScrapingError                 (base — all lib-specific exceptions inherit)
+    ├── VendorHTTPError           (transport-error or 5xx/429 retry-exhaustion)
+    ├── BlockedError              (403 / Cloudflare challenge / Akamai EVA / Distil — anti-bot)
+    ├── ParseError                (extractor couldn't find expected fields in the response)
+    └── VendorUnavailable         (alias for VendorHTTPError; intended for circuit-breaker
+                                   feeders to map "their fault" failures cleanly)
+
+Consumers wrapping the lib in their own circuit breaker typically
+trigger on ``VendorHTTPError`` / ``VendorUnavailable`` / ``BlockedError``
+but *not* on ``ParseError`` — the latter is "our bug" (parser drift),
+not "vendor down". See ``scrapper_tool.adapter`` (M7) for the generic
+Adapter Protocol that codifies this distinction.
+"""
+
+from __future__ import annotations
+
+
+class ScrapingError(Exception):
+    """Base for all scrapper-tool exceptions."""
+
+
+class VendorHTTPError(ScrapingError):
+    """Raised when all retry attempts exhaust on a retriable failure.
+
+    Non-retriable HTTP 4xx responses (other than 429) are NOT wrapped —
+    the caller sees a plain response object via the underlying client.
+    The distinction matters for circuit breakers: 4xx is "our bug",
+    5xx/429/transport is "their fault".
+    """
+
+
+class VendorUnavailable(VendorHTTPError):
+    """Alias of :class:`VendorHTTPError` for circuit-breaker call-sites.
+
+    Subclasses :class:`VendorHTTPError` so existing handlers keep
+    working; the dedicated name reads better at the breaker boundary.
+    """
+
+
+class BlockedError(ScrapingError):
+    """Raised when an anti-bot platform blocks the request.
+
+    Distinct from :class:`VendorHTTPError` because the remediation
+    differs: the breaker should NOT trip (the vendor is up, just
+    fingerprinting us). Consumer should escalate to the next ladder
+    profile or to Pattern D (Scrapling).
+    """
+
+
+class ParseError(ScrapingError):
+    """Raised when the extractor cannot find expected fields.
+
+    Indicates parser drift (vendor changed markup) or a fixture-vs-live
+    mismatch. NOT a circuit-breaker signal — re-fetching won't help.
+    """
+
+
+__all__ = [
+    "BlockedError",
+    "ParseError",
+    "ScrapingError",
+    "VendorHTTPError",
+    "VendorUnavailable",
+]