scrapenest 0.3.0__py3-none-any.whl

scrapenest/__init__.py

@@ -0,0 +1,28 @@
+from ._async_client import AsyncClient
+from ._client import Client
+from .errors import (
+    AuthenticationError,
+    ForbiddenError,
+    NotFoundError,
+    PaymentRequiredError,
+    RateLimitError,
+    ScrapeNestError,
+    ServerError,
+    UnexpectedResponseError,
+)
+
+__version__ = "0.3.0"
+
+__all__ = [
+    "Client",
+    "AsyncClient",
+    "ScrapeNestError",
+    "AuthenticationError",
+    "PaymentRequiredError",
+    "ForbiddenError",
+    "NotFoundError",
+    "RateLimitError",
+    "ServerError",
+    "UnexpectedResponseError",
+    "__version__",
+]

scrapenest/_async_client.py

@@ -0,0 +1,573 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Literal, Optional
+from urllib.parse import quote
+
+import httpx
+
+from ._http import (
+    DEFAULT_BASE_URL,
+    DEFAULT_TIMEOUT,
+    _build_headers,
+    _detect_stream_error,
+    _parse_error,
+    _wrap_network_error,
+)
+from ._place_id import normalize_place_id
+from .types.audio import AudioTranscriptResponse
+from .types.auth import MeResponse
+from .types.gas_prices import GasPricesResponse
+from .types.maps import MapsReviewsResponse
+from .types.runs import RunCreatedResponse, RunStatusResponse
+from .types.pdf import PdfResponse
+from .types.scrape import ScrapeUrlResponse
+from .types.screenshot import ScreenshotResponse
+from .types.search import (
+    AnswerResponse,
+    DeepSearchResponse,
+    SearchResponse,
+    TranslateResponse,
+)
+
+_TERMINAL = {"completed", "failed", "cancelled"}
+
+
+def _build_search_payload(
+    *,
+    query: str,
+    num_results: int,
+    language: str,
+    time_range: Optional[str],
+    categories: Optional[str],
+    include_domains: Optional[list],
+    exclude_domains: Optional[list],
+    topic: Optional[str],
+    country: Optional[str],
+    start_date: Optional[str],
+    end_date: Optional[str],
+    include_images: bool,
+    include_image_descriptions: bool,
+    image_sources: Optional[list],
+    engines: Optional[list],
+    include_usage: bool,
+    cache_ttl_seconds: Optional[int],
+) -> dict:
+    payload: dict = {"query": query, "num_results": num_results, "language": language}
+    if time_range is not None:
+        payload["time_range"] = time_range
+    if categories is not None:
+        payload["categories"] = categories
+    if include_domains is not None:
+        payload["include_domains"] = list(include_domains)
+    if exclude_domains is not None:
+        payload["exclude_domains"] = list(exclude_domains)
+    if topic is not None:
+        payload["topic"] = topic
+    if country is not None:
+        payload["country"] = country
+    if start_date is not None:
+        payload["start_date"] = start_date
+    if end_date is not None:
+        payload["end_date"] = end_date
+    if include_images:
+        payload["include_images"] = True
+    if include_image_descriptions:
+        payload["include_image_descriptions"] = True
+    if image_sources is not None:
+        payload["image_sources"] = list(image_sources)
+    if engines is not None:
+        payload["engines"] = list(engines)
+    if include_usage:
+        payload["include_usage"] = True
+    if cache_ttl_seconds is not None:
+        payload["cache_ttl_seconds"] = cache_ttl_seconds
+    return payload
+
+
+class AsyncClient:
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        extra_headers: Optional[dict[str, str]] = None,
+    ) -> None:
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+        merged: dict[str, str] = {}
+        if extra_headers:
+            merged.update({str(k): str(v) for k, v in extra_headers.items()})
+        merged.update(_build_headers(api_key))
+        self._http = httpx.AsyncClient(
+            base_url=self._base_url,
+            headers=merged,
+            timeout=timeout,
+        )
+
+    async def _request(self, method: str, path: str, **kwargs) -> dict:
+        try:
+            response = await self._http.request(method, path, **kwargs)
+        except httpx.RequestError as exc:
+            raise _wrap_network_error(exc) from exc
+        if response.is_error:
+            try:
+                body = response.json()
+            except Exception:
+                body = {}
+            raise _parse_error(response.status_code, body)
+        data = response.json()
+        stream_error = _detect_stream_error(data)
+        if stream_error is not None:
+            raise _parse_error(*stream_error)
+        return data
+
+    async def close(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncClient":
+        return self
+
+    async def __aexit__(self, *args) -> None:
+        await self.close()
+
+    async def health(self) -> dict:
+        return await self._request("GET", "/health")
+
+    async def me(self) -> MeResponse:
+        return MeResponse.from_dict(await self._request("GET", "/v1/me"))
+
+    async def scrape_url(
+        self,
+        url: str,
+        *,
+        render: str = "auto",
+        force_proxy: bool = False,
+        block_assets: bool = True,
+        extract: bool = True,
+        ai_query: Optional[str] = None,
+        ai_extract_schema: Optional[dict] = None,
+        include_usage: bool = False,
+        include_html: bool = False,
+        return_markdown: bool = False,
+        extract_rules: Optional[dict] = None,
+        wait_ms: Optional[int] = None,
+        wait_for_selector: Optional[str] = None,
+        wait_for_timeout_ms: Optional[int] = None,
+        window_width: Optional[int] = None,
+        window_height: Optional[int] = None,
+        country: Optional[str] = None,
+        device: str = "desktop",
+        block_ads: bool = False,
+        simulate_behavior: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+        session_id: Optional[str] = None,
+        cookies: Optional[dict] = None,
+        headers: Optional[dict] = None,
+    ) -> ScrapeUrlResponse:
+        payload: dict = {
+            "url": url,
+            "render": render,
+            "force_proxy": force_proxy,
+            "block_assets": block_assets,
+            "extract": extract,
+        }
+        if ai_query is not None:
+            payload["ai_query"] = ai_query
+        if ai_extract_schema is not None:
+            payload["ai_extract_schema"] = ai_extract_schema
+        if include_usage:
+            payload["include_usage"] = True
+        if include_html:
+            payload["include_html"] = True
+        if return_markdown:
+            payload["return_markdown"] = True
+        if extract_rules is not None:
+            payload["extract_rules"] = extract_rules
+        if wait_ms is not None:
+            payload["wait_ms"] = wait_ms
+        if wait_for_selector is not None:
+            payload["wait_for_selector"] = wait_for_selector
+        if wait_for_timeout_ms is not None:
+            payload["wait_for_timeout_ms"] = wait_for_timeout_ms
+        if window_width is not None:
+            payload["window_width"] = window_width
+        if window_height is not None:
+            payload["window_height"] = window_height
+        if country is not None:
+            payload["country"] = country
+        if device != "desktop":
+            payload["device"] = device
+        if block_ads:
+            payload["block_ads"] = True
+        if simulate_behavior:
+            payload["simulate_behavior"] = True
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        if session_id is not None:
+            payload["session_id"] = session_id
+        if cookies is not None:
+            payload["cookies"] = dict(cookies)
+        if headers is not None:
+            payload["headers"] = dict(headers)
+        return ScrapeUrlResponse.from_dict(await self._request("POST", "/v1/scrape/url", json=payload))
+
+    async def screenshot(
+        self,
+        url: str,
+        *,
+        viewport_width: int = 1280,
+        viewport_height: int = 720,
+        full_page: bool = False,
+        selector: Optional[str] = None,
+        format: str = "png",
+        jpeg_quality: int = 85,
+        wait_extra_ms: int = 0,
+        session_id: Optional[str] = None,
+        cache_ttl_seconds: Optional[int] = None,
+        render: str = "auto",
+        cookies: Optional[dict] = None,
+        headers: Optional[dict] = None,
+        include_usage: bool = False,
+    ) -> ScreenshotResponse:
+        payload: dict = {
+            "url": url,
+            "render": render,
+            "viewport_width": viewport_width,
+            "viewport_height": viewport_height,
+            "full_page": full_page,
+            "format": format,
+            "jpeg_quality": jpeg_quality,
+            "wait_extra_ms": wait_extra_ms,
+        }
+        if selector is not None:
+            payload["selector"] = selector
+        if session_id is not None:
+            payload["session_id"] = session_id
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        if cookies is not None:
+            payload["cookies"] = dict(cookies)
+        if headers is not None:
+            payload["headers"] = dict(headers)
+        if include_usage:
+            payload["include_usage"] = True
+        return ScreenshotResponse.from_dict(await self._request("POST", "/v1/scrape/screenshot", json=payload))
+
+    async def pdf(
+        self,
+        url: str,
+        *,
+        format: str = "Letter",
+        landscape: bool = False,
+        viewport_width: int = 1280,
+        viewport_height: int = 720,
+        wait_extra_ms: int = 0,
+        cookies: Optional[dict] = None,
+        headers: Optional[dict] = None,
+        session_id: Optional[str] = None,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+    ) -> PdfResponse:
+        payload: dict = {
+            "url": url,
+            "format": format,
+            "landscape": landscape,
+            "viewport_width": viewport_width,
+            "viewport_height": viewport_height,
+            "wait_extra_ms": wait_extra_ms,
+        }
+        if cookies is not None:
+            payload["cookies"] = dict(cookies)
+        if headers is not None:
+            payload["headers"] = dict(headers)
+        if session_id is not None:
+            payload["session_id"] = session_id
+        if include_usage:
+            payload["include_usage"] = True
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        return PdfResponse.from_dict(await self._request("POST", "/v1/scrape/pdf", json=payload))
+
+    async def search(
+        self,
+        query: str,
+        *,
+        num_results: int = 10,
+        depth: str = "balanced",
+        language: str = "en",
+        time_range: Optional[str] = None,
+        categories: Optional[str] = None,
+        include_domains: Optional[list] = None,
+        exclude_domains: Optional[list] = None,
+        topic: Optional[str] = None,
+        country: Optional[str] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        include_images: bool = False,
+        include_image_descriptions: bool = False,
+        image_sources: Optional[list] = None,
+        engines: Optional[list] = None,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+        page: int = 1,
+    ) -> SearchResponse:
+        payload = _build_search_payload(
+            query=query, num_results=num_results, language=language,
+            time_range=time_range, categories=categories,
+            include_domains=include_domains, exclude_domains=exclude_domains,
+            topic=topic, country=country, start_date=start_date, end_date=end_date,
+            include_images=include_images,
+            include_image_descriptions=include_image_descriptions,
+            image_sources=image_sources,
+            engines=engines,
+            include_usage=include_usage, cache_ttl_seconds=cache_ttl_seconds,
+        )
+        payload["depth"] = depth
+        if page != 1:
+            payload["page"] = page
+        return SearchResponse.from_dict(await self._request("POST", "/v1/search", json=payload))
+
+    async def search_deep(
+        self,
+        query: str,
+        *,
+        num_results: int = 10,
+        fetch_top: int = 5,
+        depth: str = "balanced",
+        render: str = "auto",
+        language: str = "en",
+        time_range: Optional[str] = None,
+        categories: Optional[str] = None,
+        include_domains: Optional[list] = None,
+        exclude_domains: Optional[list] = None,
+        topic: Optional[str] = None,
+        country: Optional[str] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        include_images: bool = False,
+        include_image_descriptions: bool = False,
+        image_sources: Optional[list] = None,
+        engines: Optional[list] = None,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+    ) -> DeepSearchResponse:
+        if fetch_top < 1:
+            raise ValueError("fetch_top must be >= 1 (use client.search() if you only need SERP)")
+        if fetch_top > 10:
+            raise ValueError("fetch_top must be <= 10")
+        payload = _build_search_payload(
+            query=query, num_results=num_results, language=language,
+            time_range=time_range, categories=categories,
+            include_domains=include_domains, exclude_domains=exclude_domains,
+            topic=topic, country=country, start_date=start_date, end_date=end_date,
+            include_images=include_images,
+            include_image_descriptions=include_image_descriptions,
+            image_sources=image_sources,
+            engines=engines,
+            include_usage=include_usage, cache_ttl_seconds=cache_ttl_seconds,
+        )
+        payload["fetch_top"] = fetch_top
+        payload["render"] = render
+        payload["depth"] = depth
+        return DeepSearchResponse.from_dict(await self._request("POST", "/v1/search/deep", json=payload))
+
+    async def search_answer(
+        self,
+        query: str,
+        *,
+        num_results: int = 10,
+        max_sources: int = 5,
+        depth: str = "balanced",
+        render: str = "auto",
+        language: str = "en",
+        time_range: Optional[str] = None,
+        categories: Optional[str] = None,
+        include_domains: Optional[list] = None,
+        exclude_domains: Optional[list] = None,
+        topic: Optional[str] = None,
+        country: Optional[str] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        engines: Optional[list] = None,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+    ) -> AnswerResponse:
+        payload = _build_search_payload(
+            query=query, num_results=num_results, language=language,
+            time_range=time_range, categories=categories,
+            include_domains=include_domains, exclude_domains=exclude_domains,
+            topic=topic, country=country, start_date=start_date, end_date=end_date,
+            include_images=False,
+            include_image_descriptions=False,
+            image_sources=None,
+            engines=engines,
+            include_usage=include_usage, cache_ttl_seconds=cache_ttl_seconds,
+        )
+        payload["max_sources"] = max_sources
+        payload["render"] = render
+        payload["depth"] = depth
+        return AnswerResponse.from_dict(await self._request("POST", "/v1/search/answer", json=payload))
+
+    async def translate(
+        self,
+        text: str,
+        target: str,
+        *,
+        source: str = "auto",
+        include_usage: bool = False,
+    ) -> TranslateResponse:
+        payload: dict = {"text": text, "target": target, "source": source}
+        if include_usage:
+            payload["include_usage"] = True
+        return TranslateResponse.from_dict(
+            await self._request("POST", "/v1/translate", json=payload)
+        )
+
+    async def audio_transcript(
+        self,
+        url: str,
+        *,
+        cache_ttl_seconds: Optional[int] = None,
+        language: Optional[str] = None,
+        cookies: Optional[str] = None,
+        include_usage: bool = False,
+    ) -> AudioTranscriptResponse:
+        payload: dict = {"url": url}
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        if language is not None:
+            payload["language"] = language
+        if cookies is not None:
+            payload["cookies"] = cookies
+        if include_usage:
+            payload["include_usage"] = True
+        return AudioTranscriptResponse.from_dict(
+            await self._request("POST", "/v1/audio/transcript", json=payload)
+        )
+
+    async def audio_transcript_async(
+        self,
+        url: str,
+        *,
+        cache_ttl_seconds: Optional[int] = None,
+        language: Optional[str] = None,
+        cookies: Optional[str] = None,
+        include_usage: bool = False,
+    ) -> RunCreatedResponse:
+        payload: dict = {"url": url}
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        if language is not None:
+            payload["language"] = language
+        if cookies is not None:
+            payload["cookies"] = cookies
+        if include_usage:
+            payload["include_usage"] = True
+        return RunCreatedResponse.from_dict(
+            await self._request("POST", "/v1/audio/transcript/async", json=payload)
+        )
+
+    async def maps_reviews(
+        self,
+        place_id: str,
+        *,
+        language: str = "en",
+        sort: Literal["newest", "most_relevant", "highest_rating", "lowest_rating"] = "newest",
+        max_reviews: int = 200,
+        place_url: Optional[str] = None,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+    ) -> MapsReviewsResponse:
+        payload: dict = {
+            "place_id": normalize_place_id(place_id),
+            "language": language,
+            "sort": sort,
+            "max_reviews": max_reviews,
+        }
+        if place_url is not None:
+            payload["place_url"] = place_url
+        if include_usage:
+            payload["include_usage"] = True
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        return MapsReviewsResponse.from_dict(
+            await self._request("POST", "/v1/scrape/google-maps/reviews", json=payload)
+        )
+
+    async def maps_reviews_async(
+        self,
+        place_id: str,
+        *,
+        language: str = "en",
+        sort: Literal["newest", "most_relevant", "highest_rating", "lowest_rating"] = "newest",
+        max_reviews: int = 200,
+        place_url: Optional[str] = None,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+    ) -> RunCreatedResponse:
+        payload: dict = {
+            "place_id": normalize_place_id(place_id),
+            "language": language,
+            "sort": sort,
+            "max_reviews": max_reviews,
+        }
+        if place_url is not None:
+            payload["place_url"] = place_url
+        if include_usage:
+            payload["include_usage"] = True
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        return RunCreatedResponse.from_dict(
+            await self._request("POST", "/v1/scrape/google-maps/reviews/async", json=payload)
+        )
+
+    async def gas_prices(
+        self,
+        location: str,
+        *,
+        grade: str = "regular",
+        limit: int = 20,
+        include_usage: bool = False,
+        cache_ttl_seconds: Optional[int] = None,
+    ) -> GasPricesResponse:
+        payload: dict = {
+            "location": location,
+            "grade": grade,
+            "limit": limit,
+        }
+        if include_usage:
+            payload["include_usage"] = True
+        if cache_ttl_seconds is not None:
+            payload["cache_ttl_seconds"] = cache_ttl_seconds
+        return GasPricesResponse.from_dict(
+            await self._request("POST", "/v1/scrape/gas-prices", json=payload)
+        )
+
+    async def get_run(self, run_id: str) -> RunStatusResponse:
+        return RunStatusResponse.from_dict(
+            await self._request("GET", f"/v1/scrape/runs/{quote(run_id, safe='')}")
+        )
+
+    async def cancel_run(self, run_id: str) -> RunStatusResponse:
+        return RunStatusResponse.from_dict(
+            await self._request("POST", f"/v1/scrape/runs/{quote(run_id, safe='')}/cancel")
+        )
+
+    async def wait_for_run(
+        self,
+        run_id: str,
+        *,
+        poll_interval: float = 3.0,
+        timeout: float = 300.0,
+    ) -> RunStatusResponse:
+        deadline = time.monotonic() + timeout
+        while True:
+            run = await self.get_run(run_id)
+            if run.status in _TERMINAL:
+                return run
+            if time.monotonic() >= deadline:
+                raise TimeoutError(f"Run {run_id} did not complete within {timeout}s")
+            await asyncio.sleep(poll_interval)