litescrape-sdk 0.1.0__py3-none-any.whl

litescrape_sdk/__init__.py

@@ -0,0 +1,91 @@
+"""Python SDK for the Litescrape API. Start with ``help(litescrape_sdk.scrape)``."""
+
+from ._version import __version__
+from .client import Result, akey_status, ascrape, key_status, scrape
+from .errors import (
+    APIError,
+    AuthenticationError,
+    LitescrapeError,
+    NotFoundError,
+    PaymentRequiredError,
+    RateLimitError,
+    TransportError,
+    ValidationError,
+)
+from .models import (
+    REQUEST_TYPES,
+    AnyRequest,
+    AppleMapsPlaces,
+    AppleMapsReviews,
+    BingMaps,
+    BingSearch,
+    DuckDuckGoMaps,
+    DuckDuckGoSearch,
+    GoogleAds,
+    GoogleAiMode,
+    GoogleAiOverview,
+    GoogleContributorReviews,
+    GoogleLocal,
+    GoogleMaps,
+    GoogleMapsLiveFootTraffic,
+    GoogleMapsPhoto,
+    GoogleMapsPosts,
+    GoogleMapsWebResults,
+    GoogleReviews,
+    GoogleSearch,
+    GoogleShopping,
+    GoogleShoppingProduct,
+    KeyStatus,
+    ScrapeRequest,
+    TripadvisorPlace,
+    TripadvisorReviews,
+    TripadvisorSearch,
+    YelpReviews,
+    YelpSearch,
+)
+
+__all__ = [
+    "__version__",
+    "scrape",
+    "ascrape",
+    "key_status",
+    "akey_status",
+    "Result",
+    "KeyStatus",
+    "ScrapeRequest",
+    "AnyRequest",
+    "REQUEST_TYPES",
+    "LitescrapeError",
+    "ValidationError",
+    "TransportError",
+    "APIError",
+    "AuthenticationError",
+    "PaymentRequiredError",
+    "NotFoundError",
+    "RateLimitError",
+    "GoogleSearch",
+    "GoogleAiOverview",
+    "GoogleAiMode",
+    "GoogleAds",
+    "GoogleShopping",
+    "GoogleShoppingProduct",
+    "GoogleLocal",
+    "GoogleMaps",
+    "GoogleMapsLiveFootTraffic",
+    "GoogleMapsPosts",
+    "GoogleMapsPhoto",
+    "GoogleMapsWebResults",
+    "GoogleReviews",
+    "GoogleContributorReviews",
+    "BingSearch",
+    "BingMaps",
+    "DuckDuckGoSearch",
+    "DuckDuckGoMaps",
+    "YelpSearch",
+    "YelpReviews",
+    "TripadvisorSearch",
+    "TripadvisorPlace",
+    "TripadvisorReviews",
+    "AppleMapsPlaces",
+    "AppleMapsReviews",
+]

litescrape_sdk/_runtime.py

@@ -0,0 +1,201 @@
+"""Event loop, HTTP client, concurrency gate, and retry loop behind the public functions."""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import contextlib
+import random
+import sys
+import threading
+from collections.abc import Callable, Coroutine
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+import httpx
+
+from ._version import __version__
+from .errors import APIError, LitescrapeError, TransportError, api_error_from_response
+
+DEFAULT_BASE_URL = "https://api.litescrape.com"
+DEFAULT_CONCURRENCY = 25
+SELECTOR_LOOP_CAP = 500
+BACKOFF_CAP = 30.0
+CONNECT_TIMEOUT = 10.0
+
+_RETRY_CODES = frozenset(
+    {"proxy_capacity_unavailable", "upstream_session_unavailable", "service_unavailable"}
+)
+_RETRY_TRANSPORT = (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError)
+
+T = TypeVar("T")
+_LoopEntry = tuple[asyncio.AbstractEventLoop, dict[Any, Any]]
+
+_clients: dict[int, _LoopEntry] = {}
+_semaphores: dict[int, _LoopEntry] = {}
+_sleep = asyncio.sleep
+
+
+@dataclass(slots=True)
+class Outcome:
+    data: dict[str, Any] | None = None
+    error: LitescrapeError | None = None
+    status_code: int | None = None
+    request_id: str = ""
+    attempts: int = 0
+
+
+def _per_loop(registry: dict[int, _LoopEntry]) -> dict[Any, Any]:
+    loop = asyncio.get_running_loop()
+    for key, (known, _) in list(registry.items()):
+        if known.is_closed():
+            del registry[key]
+    return registry.setdefault(id(loop), (loop, {}))[1]
+
+
+def _make_client(base_url: str) -> httpx.AsyncClient:
+    return httpx.AsyncClient(
+        base_url=base_url,
+        headers={"Accept": "application/json", "User-Agent": f"litescrape-sdk/{__version__}"},
+        limits=httpx.Limits(max_connections=None, max_keepalive_connections=None),
+        timeout=httpx.Timeout(120.0, connect=CONNECT_TIMEOUT),
+    )
+
+
+def client_for(base_url: str) -> httpx.AsyncClient:
+    clients = _per_loop(_clients)
+    client = clients.get(base_url)
+    if client is None:
+        client = clients[base_url] = _make_client(base_url)
+    return client
+
+
+def semaphore_for(api_key: str, base_url: str, limit: int) -> asyncio.Semaphore:
+    semaphores = _per_loop(_semaphores)
+    entry = semaphores.get((api_key, base_url))
+    if entry is None or entry[0] != limit:
+        entry = semaphores[(api_key, base_url)] = (limit, asyncio.Semaphore(limit))
+    return entry[1]
+
+
+def is_selector_limited(loop: asyncio.AbstractEventLoop) -> bool:
+    return sys.platform == "win32" and isinstance(loop, asyncio.SelectorEventLoop)
+
+
+def effective_limit(status_limit: int | None, override: int | None, *, selector_limited: bool) -> int:
+    limit = status_limit or DEFAULT_CONCURRENCY
+    if override is not None:
+        limit = min(limit, override)
+    if selector_limited:
+        limit = min(limit, SELECTOR_LOOP_CAP)
+    return max(1, limit)
+
+
+def _should_retry(error: LitescrapeError) -> bool:
+    if isinstance(error, TransportError):
+        return error.retryable
+    if isinstance(error, APIError):
+        status = error.status_code or 0
+        return error.retryable or status == 429 or status >= 500 or error.error_code in _RETRY_CODES
+    return False
+
+
+def _delay(attempt: int, retry_after: float | None) -> float:
+    if retry_after is not None:
+        return min(max(retry_after, 0.0), BACKOFF_CAP)
+    ceiling = min(BACKOFF_CAP, float(2 ** (attempt - 1)))
+    return random.uniform(ceiling / 2, ceiling)
+
+
+async def _attempt(
+    client: httpx.AsyncClient,
+    path: str,
+    params: dict[str, str],
+    headers: dict[str, str],
+    timeout: float,
+    outcome: Outcome,
+) -> LitescrapeError | None:
+    outcome.status_code, outcome.request_id = None, ""
+    try:
+        response = await client.get(
+            path,
+            params=params,
+            headers=headers,
+            timeout=httpx.Timeout(timeout, connect=min(CONNECT_TIMEOUT, timeout)),
+        )
+    except _RETRY_TRANSPORT as exc:
+        return TransportError(f"{type(exc).__name__}: {exc}", retryable=True, cause=exc)
+    except Exception as exc:
+        return TransportError(f"{type(exc).__name__}: {exc}", retryable=False, cause=exc)
+    outcome.status_code = response.status_code
+    outcome.request_id = response.headers.get("x-request-id", "")
+    if response.status_code != 200:
+        error = api_error_from_response(response)
+        outcome.request_id = error.request_id or outcome.request_id
+        return error
+    try:
+        data = response.json()
+    except ValueError as exc:
+        return TransportError("Response body was not JSON", retryable=True, cause=exc)
+    if not isinstance(data, dict):
+        return TransportError("Response body was not a JSON object", retryable=True)
+    outcome.data = data
+    return None
+
+
+async def request_with_retries(
+    client: httpx.AsyncClient,
+    path: str,
+    params: dict[str, str],
+    *,
+    headers: dict[str, str],
+    attempts: int,
+    timeout: float,
+    semaphore: asyncio.Semaphore | None = None,
+    gate: Callable[[], LitescrapeError | None] | None = None,
+) -> Outcome:
+    outcome = Outcome()
+    slot = semaphore if semaphore is not None else contextlib.nullcontext()
+    for attempt in range(1, attempts + 1):
+        async with slot:
+            blocked = gate() if gate is not None else None
+            if blocked is not None:
+                outcome.error = blocked
+                outcome.status_code = getattr(blocked, "status_code", None)
+                return outcome
+            outcome.attempts = attempt
+            error = await _attempt(client, path, params, headers, timeout, outcome)
+        outcome.error = error
+        if error is None:
+            return outcome
+        if attempt == attempts or not _should_retry(error):
+            return outcome
+        await _sleep(_delay(attempt, getattr(error, "retry_after", None)))
+    return outcome
+
+
+_loop_lock = threading.Lock()
+_loop: asyncio.AbstractEventLoop | None = None
+
+
+def background_loop() -> asyncio.AbstractEventLoop:
+    global _loop
+    with _loop_lock:
+        if _loop is None or _loop.is_closed():
+            loop = asyncio.ProactorEventLoop() if sys.platform == "win32" else asyncio.new_event_loop()
+            threading.Thread(target=loop.run_forever, name="litescrape-sdk", daemon=True).start()
+            _loop = loop
+        return _loop
+
+
+def run_sync(coro: Coroutine[Any, Any, T]) -> T:
+    future = asyncio.run_coroutine_threadsafe(coro, background_loop())
+    try:
+        while True:
+            try:
+                return future.result(timeout=0.25)
+            except concurrent.futures.TimeoutError:
+                continue
+    except BaseException:
+        future.cancel()
+        raise

litescrape_sdk/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

litescrape_sdk/client.py

@@ -0,0 +1,331 @@
+"""Public entry points: scrape / ascrape and key_status / akey_status."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+import pydantic
+from tqdm import tqdm
+
+from . import _runtime
+from .errors import (
+    APIError,
+    AuthenticationError,
+    LitescrapeError,
+    PaymentRequiredError,
+    TransportError,
+    ValidationError,
+)
+from .models import REQUEST_ADAPTER, REQUEST_TYPES, KeyStatus, ScrapeRequest
+
+RequestItem = Mapping[str, Any] | ScrapeRequest
+_FATAL_CODES = frozenset({"api_key_disabled"})
+
+
+@dataclass(frozen=True, slots=True)
+class Result:
+    """Outcome of one request in a batch; ``index`` is its position in the input list."""
+
+    index: int
+    request: ScrapeRequest
+    data: dict[str, Any] | None
+    error: LitescrapeError | None
+    status_code: int | None
+    request_id: str
+    attempts: int
+    elapsed: float
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None
+
+    def raise_for_error(self) -> dict[str, Any]:
+        if self.error is not None:
+            raise self.error
+        return self.data if self.data is not None else {}
+
+
+class _Batch:
+    __slots__ = ("fatal",)
+
+    def __init__(self) -> None:
+        self.fatal: APIError | None = None
+
+
+def _describe(exc: pydantic.ValidationError) -> str:
+    lines = []
+    for error in exc.errors():
+        location = ".".join(str(part) for part in error["loc"] if part not in REQUEST_TYPES)
+        message = error["msg"]
+        if error["type"] == "union_tag_not_found":
+            message += f" (valid endpoint values: {', '.join(sorted(REQUEST_TYPES))})"
+        lines.append(f"{location}: {message}" if location else message)
+    return "; ".join(lines)
+
+
+def _validate(items: Sequence[RequestItem]) -> list[ScrapeRequest]:
+    validated: list[ScrapeRequest] = []
+    problems: list[tuple[int, str]] = []
+    for index, item in enumerate(items):
+        try:
+            validated.append(REQUEST_ADAPTER.validate_python(item))
+        except pydantic.ValidationError as exc:
+            problems.append((index, _describe(exc)))
+    if problems:
+        raise ValidationError(problems)
+    return validated
+
+
+def _check_args(attempts: int, concurrency: int | None, timeout: float) -> None:
+    if attempts < 1:
+        raise ValueError("attempts must be at least 1")
+    if concurrency is not None and concurrency < 1:
+        raise ValueError("concurrency must be at least 1")
+    if timeout <= 0:
+        raise ValueError("timeout must be positive")
+
+
+def _resolve_key(api_key: str | None) -> str:
+    key = (api_key or os.environ.get("LITESCRAPE_API_KEY") or "").strip()
+    if not key:
+        raise AuthenticationError(
+            "Pass api_key or set the LITESCRAPE_API_KEY environment variable.",
+            status_code=None,
+            error_code="missing_api_key",
+        )
+    return key
+
+
+def _resolve_base_url(base_url: str | None) -> str:
+    return (base_url or os.environ.get("LITESCRAPE_API_URL") or _runtime.DEFAULT_BASE_URL).rstrip("/")
+
+
+async def _fetch_status(
+    client: httpx.AsyncClient, headers: dict[str, str], attempts: int, timeout: float
+) -> KeyStatus:
+    outcome = await _runtime.request_with_retries(
+        client, "/api/keys/status", {}, headers=headers, attempts=attempts, timeout=timeout
+    )
+    if outcome.error is not None:
+        raise outcome.error
+    try:
+        return KeyStatus.model_validate({**(outcome.data or {}), "request_id": outcome.request_id})
+    except pydantic.ValidationError as exc:
+        raise TransportError(f"Unexpected key status body: {exc}", retryable=False, cause=exc) from exc
+
+
+async def _run_one(
+    client: httpx.AsyncClient,
+    index: int,
+    request: ScrapeRequest,
+    headers: dict[str, str],
+    attempts: int,
+    timeout: float,
+    semaphore: asyncio.Semaphore,
+    batch: _Batch,
+) -> Result:
+    started = time.monotonic()
+    try:
+        outcome = await _runtime.request_with_retries(
+            client,
+            request.path,
+            request.query_params(),
+            headers=headers,
+            attempts=attempts,
+            timeout=timeout,
+            semaphore=semaphore,
+            gate=lambda: batch.fatal,
+        )
+    except asyncio.CancelledError:
+        raise
+    except Exception as exc:
+        outcome = _runtime.Outcome(
+            error=TransportError(f"Unexpected {type(exc).__name__}: {exc}", retryable=False, cause=exc)
+        )
+    error = outcome.error
+    if isinstance(error, APIError) and (error.status_code in (401, 402) or error.error_code in _FATAL_CODES):
+        batch.fatal = error
+    return Result(
+        index=index,
+        request=request,
+        data=outcome.data,
+        error=error,
+        status_code=outcome.status_code,
+        request_id=outcome.request_id,
+        attempts=outcome.attempts,
+        elapsed=time.monotonic() - started,
+    )
+
+
+async def ascrape(
+    requests: Sequence[RequestItem],
+    *,
+    api_key: str | None = None,
+    attempts: int = 5,
+    concurrency: int | None = None,
+    timeout: float = 120.0,
+    tqdm_disable: bool = False,
+    base_url: str | None = None,
+) -> list[Result]:
+    """Async form of :func:`scrape`: same arguments, results, and errors, run on the current event loop.
+
+    ``scrape`` runs this coroutine on a private background loop so it can be called from anywhere,
+    including inside a running loop; call ``ascrape`` directly from asyncio code.
+    """
+    _check_args(attempts, concurrency, timeout)
+    items = _validate(list(requests))
+    if not items:
+        return []
+    key = _resolve_key(api_key)
+    url = _resolve_base_url(base_url)
+    headers = {"Authorization": f"Bearer {key}"}
+    client = _runtime.client_for(url)
+    status = await _fetch_status(client, headers, attempts, timeout)
+    if status.remaining_calls < len(items):
+        raise PaymentRequiredError(
+            f"This batch needs {len(items)} calls but the key has {status.remaining_calls} remaining.",
+            status_code=402,
+            error_code="payment_required",
+            request_id=status.request_id,
+        )
+    limit = _runtime.effective_limit(
+        status.concurrency_limit,
+        concurrency,
+        selector_limited=_runtime.is_selector_limited(asyncio.get_running_loop()),
+    )
+    semaphore = _runtime.semaphore_for(key, url, limit)
+    batch = _Batch()
+    tasks = [
+        asyncio.create_task(_run_one(client, index, item, headers, attempts, timeout, semaphore, batch))
+        for index, item in enumerate(items)
+    ]
+    results: list[Result | None] = [None] * len(items)
+    bar = tqdm(total=len(items), disable=tqdm_disable or len(items) == 1, unit="req")
+    try:
+        for completed in asyncio.as_completed(tasks):
+            result = await completed
+            results[result.index] = result
+            bar.update(1)
+    except BaseException:
+        for task in tasks:
+            task.cancel()
+        raise
+    finally:
+        bar.close()
+    return [result for result in results if result is not None]
+
+
+def scrape(
+    requests: Sequence[RequestItem],
+    *,
+    api_key: str | None = None,
+    attempts: int = 5,
+    concurrency: int | None = None,
+    timeout: float = 120.0,
+    tqdm_disable: bool = False,
+    base_url: str | None = None,
+) -> list[Result]:
+    """Run every item in ``requests`` against the Litescrape API; one ``Result`` per item, in input order.
+
+    Each item is a dict with an ``endpoint`` key, or a typed request object::
+
+        from litescrape_sdk import GoogleMaps, GoogleSearch, scrape
+
+        results = scrape([
+            {"endpoint": "google_search", "q": "coffee grinders", "gl": "us"},
+            GoogleMaps(q="coffee", type="search", ll="@40.745,-73.988,14z"),
+        ])
+        for result in results:
+            if result.ok:
+                print(result.data["search_metadata"]["id"])
+            else:
+                print(result.error)
+
+    ``endpoint`` values are the slugs in ``REQUEST_TYPES`` (``google_search``, ``google_maps``,
+    ``google_reviews``, ``bing_search``, ``yelp_reviews``, ...); each maps to a request class whose fields
+    are that endpoint's query parameters and whose ``path`` is its route. Dicts are validated into the same
+    classes, so both forms send identical requests. Unknown parameter names, wrong types, and missing
+    required parameters are rejected locally.
+
+    Before any request is sent:
+
+    1. Every item is validated. If any item is invalid, ``ValidationError`` names each bad index and
+       nothing is sent.
+    2. The API key is resolved from ``api_key`` or the ``LITESCRAPE_API_KEY`` environment variable
+       (``AuthenticationError`` if neither is set).
+    3. ``GET /api/keys/status`` is fetched; it is not billed. ``PaymentRequiredError`` is raised if the key
+       has fewer calls remaining than the batch needs. The status also supplies the key's concurrency limit.
+
+    Requests run concurrently up to the key's ``concurrency_limit`` (25 when the key reports none), shared by
+    every call in this process for the same key. ``concurrency`` can lower that cap, never raise it. On a
+    Windows selector event loop the cap is also limited to 500. A progress bar advances as results arrive,
+    in completion order; the returned list is always in input order. ``tqdm_disable=True`` hides the bar,
+    and a single-item call shows none.
+
+    Each item is tried up to ``attempts`` times (``attempts=1`` disables retries). Retried: connection
+    errors and timeouts, HTTP 429 and 5xx, and any error the API marks ``retryable``. Not retried: 400,
+    401, 402, 403, 404, 422. Waits grow exponentially with jitter from about 1 s, capped at 30 s, or follow
+    the API's ``Retry-After``. Only a 200 response is billed; a retry is a new call, so if an attempt
+    succeeded server-side but its response was lost in transit, both calls are billed. ``timeout`` is per
+    attempt in seconds and defaults to 120, above the API's own 90 s deadline.
+
+    Per-item failures never raise. ``Result.ok`` is False and ``Result.error`` holds the exception: an
+    ``APIError`` subclass with ``status_code``, ``error_code``, and ``request_id`` for API envelopes
+    (``NotFoundError`` for a 404, for example Google AI Overview when no overview exists), or
+    ``TransportError`` when no usable response arrived. ``Result.raise_for_error()`` raises it or returns
+    ``Result.data``. A 401, 402, or disabled-key error seen mid-batch is copied to every item not yet
+    started, without further requests.
+
+    Ctrl-C (or cancelling the task) cancels in-flight requests; a request the API had already completed
+    is still billed.
+
+    Args:
+        requests: dicts with an ``endpoint`` key, or request objects such as ``GoogleSearch(q=...)``.
+        api_key: bearer key; defaults to ``LITESCRAPE_API_KEY``.
+        attempts: total tries per item, at least 1.
+        concurrency: optional lower cap on in-flight requests, at least 1.
+        timeout: seconds per attempt.
+        tqdm_disable: hide the progress bar.
+        base_url: API origin; defaults to ``LITESCRAPE_API_URL`` or ``https://api.litescrape.com``.
+
+    Returns:
+        ``list[Result]`` aligned with ``requests``.
+
+    Raises:
+        ValidationError, AuthenticationError, PaymentRequiredError, APIError, TransportError: before any
+        scrape request is sent, as described above. ``ValueError`` for out-of-range arguments.
+    """
+    return _runtime.run_sync(
+        ascrape(
+            requests,
+            api_key=api_key,
+            attempts=attempts,
+            concurrency=concurrency,
+            timeout=timeout,
+            tqdm_disable=tqdm_disable,
+            base_url=base_url,
+        )
+    )
+
+
+async def akey_status(api_key: str | None = None, *, base_url: str | None = None) -> KeyStatus:
+    """Async form of :func:`key_status`."""
+    key = _resolve_key(api_key)
+    url = _resolve_base_url(base_url)
+    client = _runtime.client_for(url)
+    return await _fetch_status(client, {"Authorization": f"Bearer {key}"}, attempts=5, timeout=120.0)
+
+
+def key_status(api_key: str | None = None, *, base_url: str | None = None) -> KeyStatus:
+    """Read the key's balance and limits from ``GET /api/keys/status``; the call is not billed.
+
+    Returns a ``KeyStatus`` with ``remaining_calls``, ``concurrency_limit``, ``status``, and the other
+    fields the API reports. Raises ``AuthenticationError`` for a missing or invalid key.
+    """
+    return _runtime.run_sync(akey_status(api_key, base_url=base_url))