scrapely-python 0.1.0__py3-none-any.whl

scrapely/storage/api_client.py

@@ -0,0 +1,350 @@
+"""
+Base API client for Scrapely storage operations.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+from typing import Any, Dict, Optional, Union
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class ApiClientError(Exception):
+    """HTTP error from the Scrapely API."""
+
+    def __init__(self, message: str, status_code: int, body: Any = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+
+def unwrap_data(value: Any) -> Any:
+    """Unwrap Apify-style v2 response envelopes ({ data: ... })."""
+    if isinstance(value, list):
+        return value
+    if isinstance(value, dict) and 'data' in value:
+        return value['data']
+    return value
+
+
+class AdaptiveRateLimiter:
+    """Adaptive throttling based on X-RateLimit-* headers (TS AdaptiveRateLimiter)."""
+
+    def __init__(self) -> None:
+        self.limit = 1000
+        self.remaining = 1000
+        self.reset_at = 0.0
+        self.low_water_mark = 50
+        self.min_delay = 0.05
+        self.max_delay = 1.0
+        self.last_request_time = 0.0
+        self._rate_limit_errors: Dict[int, int] = {}
+
+    def update_from_headers(self, headers: httpx.Headers) -> None:
+        limit_header = headers.get('X-RateLimit-Limit')
+        remaining_header = headers.get('X-RateLimit-Remaining')
+        reset_header = headers.get('X-RateLimit-Reset')
+        if limit_header:
+            try:
+                self.limit = int(limit_header) or 600
+            except ValueError:
+                pass
+        if remaining_header:
+            try:
+                self.remaining = int(remaining_header)
+            except ValueError:
+                pass
+        if reset_header:
+            try:
+                self.reset_at = float(reset_header) * 1000
+            except ValueError:
+                pass
+
+    async def wait_for_slot(self) -> None:
+        now = time.time() * 1000
+        if now >= self.reset_at:
+            self.remaining = self.limit
+
+        delay = self.min_delay
+        if self.remaining < self.low_water_mark:
+            ratio = self.remaining / self.low_water_mark if self.low_water_mark else 0
+            delay = min(self.max_delay, self.min_delay + (1 - ratio) * 0.5)
+        elif self.remaining < self.limit * 0.2:
+            delay = 0.1
+
+        time_since_last = (now - self.last_request_time) / 1000
+        if time_since_last < delay:
+            await asyncio.sleep(delay - time_since_last)
+
+        self.last_request_time = time.time() * 1000
+
+    async def handle_rate_limited(self, response: httpx.Response) -> None:
+        self._rate_limit_errors[429] = self._rate_limit_errors.get(429, 0) + 1
+        retry_after = response.headers.get('Retry-After')
+        wait_ms = 1000.0
+        if retry_after:
+            try:
+                wait_ms = float(retry_after) * 1000
+            except ValueError:
+                try:
+                    from email.utils import parsedate_to_datetime
+
+                    retry_date = parsedate_to_datetime(retry_after).timestamp() * 1000
+                    wait_ms = max(0.0, retry_date - time.time() * 1000)
+                except Exception:
+                    wait_ms = 1000.0
+
+        wait_ms = wait_ms + random.random() * 100
+        logger.warning('[RateLimiter] Rate limited, waiting %.0fms before retry', wait_ms)
+        await asyncio.sleep(wait_ms / 1000)
+
+    def get_rate_limit_errors(self) -> Dict[int, int]:
+        return dict(self._rate_limit_errors)
+
+
+class ApiClient:
+    """Base HTTP client for Scrapely API operations.
+
+    Handles authentication, rate limiting, and error handling for all API calls.
+    """
+
+    _rate_limiter = AdaptiveRateLimiter()
+    _max_retries = 3
+
+    def __init__(
+        self,
+        api_url: str,
+        token: str,
+        run_id: str,
+        timeout: float = 30.0,
+        transport: Optional[httpx.AsyncBaseTransport] = None,
+    ):
+        self.api_url = api_url.rstrip('/')
+        self.token = token
+        self.run_id = run_id
+        self.timeout = timeout
+        self._transport = transport
+        self._client: Optional[httpx.AsyncClient] = None
+
+    async def __aenter__(self) -> 'ApiClient':
+        await self._ensure_client()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        await self.close()
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        if self._client is None or self._client.is_closed:
+            kwargs: Dict[str, Any] = {
+                'timeout': self.timeout,
+                'headers': {
+                    'Authorization': f'Bearer {self.token}',
+                    'X-Scrapely-Run-Id': self.run_id,
+                },
+            }
+            if self._transport is not None:
+                kwargs['transport'] = self._transport
+            self._client = httpx.AsyncClient(**kwargs)
+        return self._client
+
+    async def close(self) -> None:
+        if self._client and not self._client.is_closed:
+            await self._client.aclose()
+            self._client = None
+
+    def get_rate_limit_errors(self) -> Dict[int, int]:
+        return self._rate_limiter.get_rate_limit_errors()
+
+    def _url(self, path: str) -> str:
+        if path.startswith('http://') or path.startswith('https://'):
+            return path
+        if not path.startswith('/'):
+            path = f'/{path}'
+        return f'{self.api_url}{path}'
+
+    async def _raise_for_status(self, response: httpx.Response) -> None:
+        if response.is_success:
+            return
+        body: Any = None
+        message = response.reason_phrase
+        try:
+            body = response.json()
+            if isinstance(body, dict):
+                error = body.get('error') or {}
+                if isinstance(error, dict) and error.get('message'):
+                    message = error['message']
+                elif body.get('message'):
+                    message = body['message']
+        except Exception:
+            body = response.text
+            if body:
+                message = body
+        raise ApiClientError(message, response.status_code, body)
+
+    async def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Optional[Dict[str, Any]] = None,
+        json_data: Any = None,
+        content: Optional[Union[bytes, str]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        raise_for_status: bool = True,
+        do_not_retry_timeouts: bool = False,
+    ) -> httpx.Response:
+        """Make an HTTP request with adaptive rate limiting and 429 retries."""
+        client = await self._ensure_client()
+        url = self._url(path)
+        logger.debug('%s %s params=%s', method, url, params)
+
+        request_headers = dict(headers or {})
+        if json_data is not None and 'Content-Type' not in request_headers:
+            request_headers['Content-Type'] = 'application/json'
+
+        last_error: Optional[Exception] = None
+        for attempt in range(self._max_retries + 1):
+            await self._rate_limiter.wait_for_slot()
+            try:
+                response = await client.request(
+                    method,
+                    url,
+                    params=params,
+                    json=json_data,
+                    content=content,
+                    headers=request_headers or None,
+                )
+            except httpx.TimeoutException as exc:
+                last_error = exc
+                if do_not_retry_timeouts or attempt >= self._max_retries:
+                    raise
+                await asyncio.sleep(0.25 * (attempt + 1))
+                continue
+
+            self._rate_limiter.update_from_headers(response.headers)
+
+            if response.status_code == 429 and attempt < self._max_retries:
+                await self._rate_limiter.handle_rate_limited(response)
+                continue
+
+            if raise_for_status:
+                await self._raise_for_status(response)
+            return response
+
+        if last_error is not None:
+            raise last_error
+        raise ApiClientError('Request failed after retries', 429)
+
+    async def get(
+        self,
+        path: str,
+        params: Optional[Dict[str, Any]] = None,
+        *,
+        raise_for_status: bool = True,
+    ) -> httpx.Response:
+        return await self.request('GET', path, params=params, raise_for_status=raise_for_status)
+
+    async def head(
+        self,
+        path: str,
+        params: Optional[Dict[str, Any]] = None,
+        *,
+        raise_for_status: bool = True,
+    ) -> httpx.Response:
+        return await self.request('HEAD', path, params=params, raise_for_status=raise_for_status)
+
+    async def post(
+        self,
+        path: str,
+        json_data: Any = None,
+        content: Optional[Union[bytes, str]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        *,
+        raise_for_status: bool = True,
+    ) -> httpx.Response:
+        return await self.request(
+            'POST',
+            path,
+            params=params,
+            json_data=json_data,
+            content=content,
+            headers=headers,
+            raise_for_status=raise_for_status,
+        )
+
+    async def put(
+        self,
+        path: str,
+        json_data: Any = None,
+        content: Optional[Union[bytes, str]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        *,
+        raise_for_status: bool = True,
+    ) -> httpx.Response:
+        return await self.request(
+            'PUT',
+            path,
+            params=params,
+            json_data=json_data,
+            content=content,
+            headers=headers,
+            raise_for_status=raise_for_status,
+        )
+
+    async def delete(
+        self,
+        path: str,
+        params: Optional[Dict[str, Any]] = None,
+        json_data: Any = None,
+        *,
+        raise_for_status: bool = True,
+    ) -> httpx.Response:
+        return await self.request(
+            'DELETE',
+            path,
+            params=params,
+            json_data=json_data,
+            raise_for_status=raise_for_status,
+        )
+
+    async def get_json(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        response = await self.get(path, params=params)
+        return unwrap_data(response.json())
+
+    async def post_json(
+        self,
+        path: str,
+        data: Any = None,
+        params: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Any:
+        response = await self.post(path, json_data=data, params=params, headers=headers)
+        if not response.content:
+            return {}
+        try:
+            return unwrap_data(response.json())
+        except Exception:
+            return {}
+
+    async def put_json(
+        self,
+        path: str,
+        data: Any = None,
+        params: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Any:
+        response = await self.put(path, json_data=data, params=params, headers=headers)
+        if not response.content:
+            return {}
+        try:
+            return unwrap_data(response.json())
+        except Exception:
+            return {}