clawfetch 0.1.0__py3-none-any.whl

clawfetch/__init__.py

@@ -0,0 +1,17 @@
+"""ClawFetch — Web Intelligence API for AI Agents (x402-native)"""
+
+from .client import ClawFetch, RetryOptions
+from .async_client import AsyncClawFetch
+from .errors import ApiError, ClawFetchError, NetworkError, PaymentError, RateLimitError
+
+__all__ = [
+    "ClawFetch",
+    "AsyncClawFetch",
+    "RetryOptions",
+    "ClawFetchError",
+    "PaymentError",
+    "NetworkError",
+    "RateLimitError",
+    "ApiError",
+]
+__version__ = "0.1.0"

clawfetch/async_client.py

@@ -0,0 +1,451 @@
+"""ClawFetch Async Python SDK — async/await support using httpx.AsyncClient."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import logging
+import os
+import random
+import time
+from typing import Any, Dict, List, Optional
+
+import httpx
+from eth_account import Account
+from eth_account.messages import encode_typed_data
+
+from .client import RetryOptions, BASE_URL, _RETRYABLE_STATUS_CODES
+from .errors import ApiError, ClawFetchError, NetworkError, PaymentError, RateLimitError
+
+logger = logging.getLogger("clawfetch.async")
+
+
+class AsyncClawFetch:
+    """Async client for the ClawFetch Web Intelligence API.
+
+    Mirrors all methods from the sync ``ClawFetch`` client, but uses
+    ``httpx.AsyncClient`` and ``async/await`` for non-blocking I/O.
+
+    Handles the full x402 payment flow asynchronously:
+    1. Make request → get 402 with payment requirements
+    2. Sign EIP-3009 gasless USDC transfer on Base
+    3. Retry with PAYMENT-SIGNATURE header → get data
+
+    Usage::
+
+        import asyncio
+        from clawfetch import AsyncClawFetch
+
+        async def main():
+            async with AsyncClawFetch(private_key="0x...") as cf:
+                btc = await cf.extract("https://coingecko.com/en/coins/bitcoin")
+                print(btc["data"])
+
+        asyncio.run(main())
+    """
+
+    def __init__(
+        self,
+        private_key: str,
+        base_url: str = BASE_URL,
+        timeout: float = 30.0,
+        retry: RetryOptions | bool | None = None,
+        debug: bool = False,
+    ):
+        """Initialize AsyncClawFetch client.
+
+        Args:
+            private_key: Ethereum private key (hex string with 0x prefix).
+            base_url: API base URL. Defaults to https://api.clawfetch.ai.
+            timeout: Request timeout in seconds. Defaults to 30.
+            retry: Retry configuration. Pass False to disable, True or None for defaults,
+                   or a RetryOptions instance for custom config.
+            debug: Enable debug logging.
+        """
+        self._account = Account.from_key(private_key)
+        self._base_url = base_url.rstrip("/")
+        self._timeout = timeout
+        self._client = httpx.AsyncClient(timeout=timeout)
+
+        # Configure retry
+        if retry is False:
+            self._retry: RetryOptions | None = None
+        elif retry is True or retry is None:
+            self._retry = RetryOptions()
+        elif isinstance(retry, RetryOptions):
+            self._retry = retry
+        else:
+            self._retry = RetryOptions()
+
+        if debug:
+            logger.setLevel(logging.DEBUG)
+            if not logger.handlers:
+                handler = logging.StreamHandler()
+                handler.setFormatter(
+                    logging.Formatter("[clawfetch.async] %(levelname)s %(message)s")
+                )
+                logger.addHandler(handler)
+
+    @property
+    def address(self) -> str:
+        """Wallet address derived from the private key."""
+        return self._account.address
+
+    async def close(self) -> None:
+        """Close the underlying HTTP client."""
+        await self._client.aclose()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()
+
+    # ─── Public endpoints ──────────────────────────────────────
+
+    async def fetch(self, url: str, *, max_chars: int | None = None) -> dict:
+        """Fetch a URL as clean markdown ($0.001).
+
+        Args:
+            url: URL to fetch.
+            max_chars: Maximum characters to return.
+
+        Returns:
+            Dict with url, title, content, contentType fields.
+
+        Raises:
+            PaymentError: If x402 payment fails.
+            RateLimitError: If rate limited (429).
+            ApiError: If server returns 4xx/5xx.
+            NetworkError: If connection fails.
+        """
+        body: dict = {"url": url}
+        if max_chars:
+            body["maxChars"] = max_chars
+        return await self._paid_post("/fetch", body)
+
+    async def render(self, url: str, *, max_chars: int | None = None) -> dict:
+        """Render JS-heavy page with stealth browser ($0.002).
+
+        Args:
+            url: URL to render.
+            max_chars: Maximum characters to return.
+
+        Returns:
+            Dict with url, title, content fields.
+        """
+        body: dict = {"url": url}
+        if max_chars:
+            body["maxChars"] = max_chars
+        return await self._paid_post("/render", body)
+
+    async def extract(self, url: str) -> dict:
+        """Extract structured data from supported URL ($0.003).
+
+        Args:
+            url: URL to extract from (must match a supported extractor).
+
+        Returns:
+            Dict with url, extractor, data fields.
+        """
+        return await self._paid_post("/extract", {"url": url})
+
+    async def research(self, topic: str, *, sources: int | None = None) -> dict:
+        """Multi-source research on a topic ($0.01).
+
+        Args:
+            topic: Research topic/query.
+            sources: Number of sources to consult.
+
+        Returns:
+            Dict with topic, summary, sources fields.
+        """
+        body: dict = {"topic": topic}
+        if sources:
+            body["sources"] = sources
+        return await self._paid_post("/research", body)
+
+    async def domains_check(self, domains: list[str]) -> dict:
+        """Check domain availability ($0.002).
+
+        Args:
+            domains: List of domain names to check.
+
+        Returns:
+            Dict with domains list, each having domain and available fields.
+        """
+        return await self._paid_post("/domains/check", {"domains": domains})
+
+    async def domains_suggest(self, query: str, *, tlds: list[str] | None = None) -> dict:
+        """Generate domain suggestions ($0.002).
+
+        Args:
+            query: Topic or keyword for domain suggestions.
+            tlds: Preferred TLDs (e.g., [".ai", ".dev"]).
+
+        Returns:
+            Dict with query and suggestions list.
+        """
+        body: dict = {"query": query}
+        if tlds:
+            body["tlds"] = tlds
+        return await self._paid_post("/domains/suggest", body)
+
+    async def extractors(self) -> list[dict]:
+        """List available extractors ($0.001).
+
+        Returns:
+            List of extractor dicts with name, domains, description, fields.
+        """
+        resp = await self._paid_request("GET", "/extractors")
+        return resp.get("extractors", [])
+
+    async def health(self) -> dict:
+        """Check service health (free, no payment required).
+
+        Returns:
+            Dict with status, service, version fields.
+
+        Raises:
+            NetworkError: If connection to API fails.
+        """
+        try:
+            r = await self._client.get(f"{self._base_url}/health")
+            r.raise_for_status()
+            return r.json()
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            raise NetworkError(
+                f"Health check failed: {exc}", "/health", cause=exc
+            ) from exc
+
+    # ─── Retry engine ──────────────────────────────────────────
+
+    def _should_retry(self, status_code: int) -> bool:
+        """Determine if a status code is retryable."""
+        return status_code in _RETRYABLE_STATUS_CODES
+
+    def _get_retry_delay_ms(self, attempt: int, retry_after_ms: int | None = None) -> int:
+        """Calculate delay for a retry attempt with exponential backoff + jitter."""
+        if self._retry is None:
+            return 0
+
+        if retry_after_ms and retry_after_ms > 0:
+            return retry_after_ms
+
+        delay = self._retry.initial_delay_ms * (
+            self._retry.backoff_multiplier ** attempt
+        )
+        # Add jitter (±25%)
+        jitter = delay * 0.25
+        delay = delay + random.uniform(-jitter, jitter)
+        return min(int(delay), self._retry.max_delay_ms)
+
+    def _parse_retry_after(self, headers: httpx.Headers) -> int | None:
+        """Parse Retry-After header into milliseconds."""
+        val = headers.get("retry-after")
+        if val is None:
+            return None
+        try:
+            return int(float(val) * 1000)
+        except (ValueError, TypeError):
+            return None
+
+    # ─── x402 payment flow with retry ──────────────────────────
+
+    async def _paid_post(self, path: str, body: dict) -> dict:
+        return await self._paid_request("POST", path, json_body=body)
+
+    async def _paid_request(
+        self, method: str, path: str, json_body: dict | None = None
+    ) -> dict:
+        url = f"{self._base_url}{path}"
+        kwargs: dict = {}
+        if json_body is not None:
+            kwargs["json"] = json_body
+
+        max_attempts = (self._retry.max_retries + 1) if self._retry else 1
+        last_error: Exception | None = None
+
+        for attempt in range(max_attempts):
+            try:
+                logger.debug(
+                    "Request %s %s (attempt %d/%d)", method, path, attempt + 1, max_attempts
+                )
+
+                # First request — expect 402
+                resp = await self._client.request(method, url, **kwargs)
+
+                if resp.status_code == 402:
+                    # x402 payment flow
+                    payment_required = self._parse_payment_required(resp)
+                    payment_payload = self._create_payment_payload(payment_required)
+                    encoded = base64.b64encode(
+                        json.dumps(payment_payload).encode()
+                    ).decode()
+
+                    # Retry with payment header
+                    headers = {"PAYMENT-SIGNATURE": encoded}
+                    resp = await self._client.request(method, url, headers=headers, **kwargs)
+
+                # Check for retryable errors
+                if resp.status_code == 429:
+                    retry_after_ms = self._parse_retry_after(resp.headers)
+                    if attempt < max_attempts - 1 and self._retry:
+                        delay_ms = self._get_retry_delay_ms(attempt, retry_after_ms)
+                        logger.debug(
+                            "Rate limited (429), retrying in %dms", delay_ms
+                        )
+                        await asyncio.sleep(delay_ms / 1000)
+                        continue
+                    raise RateLimitError(
+                        f"Rate limited on {path}",
+                        endpoint=path,
+                        retry_after_ms=retry_after_ms,
+                    )
+
+                if resp.status_code >= 500 and self._should_retry(resp.status_code):
+                    if attempt < max_attempts - 1 and self._retry:
+                        delay_ms = self._get_retry_delay_ms(attempt)
+                        logger.debug(
+                            "Server error %d, retrying in %dms",
+                            resp.status_code,
+                            delay_ms,
+                        )
+                        await asyncio.sleep(delay_ms / 1000)
+                        continue
+                    # Final attempt — raise
+                    try:
+                        body_text = resp.json().get("error", resp.text)
+                    except Exception:
+                        body_text = str(resp.status_code)
+                    raise ApiError(
+                        f"Server error on {path}: {body_text}",
+                        resp.status_code,
+                        endpoint=path,
+                    )
+
+                if resp.status_code >= 400:
+                    # Non-retryable client errors
+                    try:
+                        body_text = resp.json().get("error", str(resp.status_code))
+                    except Exception:
+                        body_text = str(resp.status_code)
+                    raise ApiError(
+                        f"API error on {path}: {body_text}",
+                        resp.status_code,
+                        endpoint=path,
+                    )
+
+                return resp.json()
+
+            except (PaymentError, RateLimitError, ApiError):
+                raise
+
+            except (httpx.ConnectError, httpx.TimeoutException, httpx.ReadTimeout) as exc:
+                last_error = exc
+                if attempt < max_attempts - 1 and self._retry:
+                    delay_ms = self._get_retry_delay_ms(attempt)
+                    logger.debug(
+                        "Network error (%s), retrying in %dms",
+                        type(exc).__name__,
+                        delay_ms,
+                    )
+                    await asyncio.sleep(delay_ms / 1000)
+                    continue
+                raise NetworkError(
+                    f"Network error on {path}: {exc}",
+                    endpoint=path,
+                    cause=exc,
+                ) from exc
+
+        # Should never reach here, but just in case
+        raise NetworkError(
+            f"All {max_attempts} attempts failed for {path}",
+            endpoint=path,
+            cause=last_error,
+        )
+
+    # ─── x402 helpers (same as sync — signing is CPU-bound) ───
+
+    def _parse_payment_required(self, resp: httpx.Response) -> dict:
+        """Parse x402 payment requirements from 402 response."""
+        header = resp.headers.get("payment-required") or resp.headers.get(
+            "PAYMENT-REQUIRED"
+        )
+        if header:
+            return json.loads(base64.b64decode(header))
+
+        body = resp.json()
+        if "x402Version" in body:
+            return body
+
+        raise ValueError("Cannot parse x402 payment requirements from 402 response")
+
+    def _create_payment_payload(self, payment_required: dict) -> dict:
+        """Create EIP-3009 TransferWithAuthorization payload."""
+        if "accepts" in payment_required:
+            req = payment_required["accepts"][0]
+        else:
+            req = payment_required
+
+        now = int(time.time())
+        nonce = os.urandom(32)
+
+        authorization = {
+            "from": self._account.address,
+            "to": req["payTo"],
+            "value": str(req.get("maxAmountRequired", req.get("amount", "0"))),
+            "validAfter": str(now - 600),
+            "validBefore": str(now + req.get("maxTimeoutSeconds", 300)),
+            "nonce": "0x" + nonce.hex(),
+        }
+
+        extra = req.get("extra", {})
+        token_name = extra.get("name", "USD Coin")
+        token_version = extra.get("version", "2")
+        chain_id = int(req["network"].split(":")[1])
+        token_address = req["asset"]
+
+        domain = {
+            "name": token_name,
+            "version": token_version,
+            "chainId": chain_id,
+            "verifyingContract": token_address,
+        }
+
+        types = {
+            "TransferWithAuthorization": [
+                {"name": "from", "type": "address"},
+                {"name": "to", "type": "address"},
+                {"name": "value", "type": "uint256"},
+                {"name": "validAfter", "type": "uint256"},
+                {"name": "validBefore", "type": "uint256"},
+                {"name": "nonce", "type": "bytes32"},
+            ],
+        }
+
+        message = {
+            "from": authorization["from"],
+            "to": authorization["to"],
+            "value": int(authorization["value"]),
+            "validAfter": int(authorization["validAfter"]),
+            "validBefore": int(authorization["validBefore"]),
+            "nonce": bytes.fromhex(authorization["nonce"][2:]),
+        }
+
+        signed = self._account.sign_typed_data(
+            domain_data=domain,
+            message_types=types,
+            message_data=message,
+        )
+
+        return {
+            "x402Version": 2,
+            "payload": {
+                "authorization": authorization,
+                "signature": signed.signature.hex()
+                if isinstance(signed.signature, bytes)
+                else str(signed.signature),
+            },
+        }

clawfetch/client.py

@@ -0,0 +1,478 @@
+"""ClawFetch Python SDK — handles x402 payment flow automatically."""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import httpx
+from eth_account import Account
+from eth_account.messages import encode_typed_data
+
+from .errors import ApiError, ClawFetchError, NetworkError, PaymentError, RateLimitError
+
+BASE_URL = "https://api.clawfetch.ai"
+
+logger = logging.getLogger("clawfetch")
+
+
+@dataclass
+class RetryOptions:
+    """Configuration for automatic retry with exponential backoff."""
+
+    max_retries: int = 3
+    """Maximum number of retry attempts."""
+
+    initial_delay_ms: int = 500
+    """Initial delay in milliseconds before first retry."""
+
+    max_delay_ms: int = 10_000
+    """Maximum delay in milliseconds between retries."""
+
+    backoff_multiplier: float = 2.0
+    """Multiplier for exponential backoff."""
+
+
+# Statuses that are safe to retry
+_RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})
+
+
+class ClawFetch:
+    """Client for the ClawFetch Web Intelligence API.
+
+    Handles the full x402 payment flow:
+    1. Make request → get 402 with payment requirements
+    2. Sign EIP-3009 gasless USDC transfer on Base
+    3. Retry with PAYMENT-SIGNATURE header → get data
+
+    Features:
+    - Automatic x402 payment signing (EIP-3009 TransferWithAuthorization)
+    - Configurable retry with exponential backoff + jitter
+    - Typed error hierarchy (PaymentError, NetworkError, RateLimitError, ApiError)
+    - Configurable timeout
+    - Debug logging
+
+    Usage::
+
+        from clawfetch import ClawFetch
+
+        cf = ClawFetch(private_key="0x...")
+        btc = cf.extract("https://coingecko.com/en/coins/bitcoin")
+        print(btc["data"])
+    """
+
+    def __init__(
+        self,
+        private_key: str,
+        base_url: str = BASE_URL,
+        timeout: float = 30.0,
+        retry: RetryOptions | bool | None = None,
+        debug: bool = False,
+    ):
+        """Initialize ClawFetch client.
+
+        Args:
+            private_key: Ethereum private key (hex string with 0x prefix).
+            base_url: API base URL. Defaults to https://api.clawfetch.ai.
+            timeout: Request timeout in seconds. Defaults to 30.
+            retry: Retry configuration. Pass False to disable, True or None for defaults,
+                   or a RetryOptions instance for custom config.
+            debug: Enable debug logging.
+        """
+        self._account = Account.from_key(private_key)
+        self._base_url = base_url.rstrip("/")
+        self._timeout = timeout
+        self._client = httpx.Client(timeout=timeout)
+
+        # Configure retry
+        if retry is False:
+            self._retry: RetryOptions | None = None
+        elif retry is True or retry is None:
+            self._retry = RetryOptions()
+        elif isinstance(retry, RetryOptions):
+            self._retry = retry
+        else:
+            self._retry = RetryOptions()
+
+        if debug:
+            logger.setLevel(logging.DEBUG)
+            if not logger.handlers:
+                handler = logging.StreamHandler()
+                handler.setFormatter(
+                    logging.Formatter("[clawfetch] %(levelname)s %(message)s")
+                )
+                logger.addHandler(handler)
+
+    @property
+    def address(self) -> str:
+        """Wallet address derived from the private key."""
+        return self._account.address
+
+    def close(self) -> None:
+        """Close the underlying HTTP client."""
+        self._client.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    # ─── Public endpoints ──────────────────────────────────────
+
+    def fetch(self, url: str, *, max_chars: int | None = None) -> dict:
+        """Fetch a URL as clean markdown ($0.001).
+
+        Args:
+            url: URL to fetch.
+            max_chars: Maximum characters to return.
+
+        Returns:
+            Dict with url, title, content, contentType fields.
+
+        Raises:
+            PaymentError: If x402 payment fails.
+            RateLimitError: If rate limited (429).
+            ApiError: If server returns 4xx/5xx.
+            NetworkError: If connection fails.
+        """
+        body: dict = {"url": url}
+        if max_chars:
+            body["maxChars"] = max_chars
+        return self._paid_post("/fetch", body)
+
+    def render(self, url: str, *, max_chars: int | None = None) -> dict:
+        """Render JS-heavy page with stealth browser ($0.002).
+
+        Args:
+            url: URL to render.
+            max_chars: Maximum characters to return.
+
+        Returns:
+            Dict with url, title, content fields.
+        """
+        body: dict = {"url": url}
+        if max_chars:
+            body["maxChars"] = max_chars
+        return self._paid_post("/render", body)
+
+    def extract(self, url: str) -> dict:
+        """Extract structured data from supported URL ($0.003).
+
+        Args:
+            url: URL to extract from (must match a supported extractor).
+
+        Returns:
+            Dict with url, extractor, data fields.
+        """
+        return self._paid_post("/extract", {"url": url})
+
+    def research(self, topic: str, *, sources: int | None = None) -> dict:
+        """Multi-source research on a topic ($0.01).
+
+        Args:
+            topic: Research topic/query.
+            sources: Number of sources to consult.
+
+        Returns:
+            Dict with topic, summary, sources fields.
+        """
+        body: dict = {"topic": topic}
+        if sources:
+            body["sources"] = sources
+        return self._paid_post("/research", body)
+
+    def domains_check(self, domains: list[str]) -> dict:
+        """Check domain availability ($0.002).
+
+        Args:
+            domains: List of domain names to check.
+
+        Returns:
+            Dict with domains list, each having domain and available fields.
+        """
+        return self._paid_post("/domains/check", {"domains": domains})
+
+    def domains_suggest(self, query: str, *, tlds: list[str] | None = None) -> dict:
+        """Generate domain suggestions ($0.002).
+
+        Args:
+            query: Topic or keyword for domain suggestions.
+            tlds: Preferred TLDs (e.g., [".ai", ".dev"]).
+
+        Returns:
+            Dict with query and suggestions list.
+        """
+        body: dict = {"query": query}
+        if tlds:
+            body["tlds"] = tlds
+        return self._paid_post("/domains/suggest", body)
+
+    def extractors(self) -> list[dict]:
+        """List available extractors ($0.001).
+
+        Returns:
+            List of extractor dicts with name, domains, description, fields.
+        """
+        resp = self._paid_request("GET", "/extractors")
+        return resp.get("extractors", [])
+
+    def health(self) -> dict:
+        """Check service health (free, no payment required).
+
+        Returns:
+            Dict with status, service, version fields.
+
+        Raises:
+            NetworkError: If connection to API fails.
+        """
+        try:
+            r = self._client.get(f"{self._base_url}/health")
+            r.raise_for_status()
+            return r.json()
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            raise NetworkError(
+                f"Health check failed: {exc}", "/health", cause=exc
+            ) from exc
+
+    # ─── Retry engine ──────────────────────────────────────────
+
+    def _should_retry(self, status_code: int) -> bool:
+        """Determine if a status code is retryable."""
+        return status_code in _RETRYABLE_STATUS_CODES
+
+    def _get_retry_delay_ms(self, attempt: int, retry_after_ms: int | None = None) -> int:
+        """Calculate delay for a retry attempt with exponential backoff + jitter."""
+        if self._retry is None:
+            return 0
+
+        if retry_after_ms and retry_after_ms > 0:
+            return retry_after_ms
+
+        delay = self._retry.initial_delay_ms * (
+            self._retry.backoff_multiplier ** attempt
+        )
+        # Add jitter (±25%)
+        jitter = delay * 0.25
+        delay = delay + random.uniform(-jitter, jitter)
+        return min(int(delay), self._retry.max_delay_ms)
+
+    def _parse_retry_after(self, headers: httpx.Headers) -> int | None:
+        """Parse Retry-After header into milliseconds."""
+        val = headers.get("retry-after")
+        if val is None:
+            return None
+        try:
+            return int(float(val) * 1000)
+        except (ValueError, TypeError):
+            return None
+
+    # ─── x402 payment flow with retry ──────────────────────────
+
+    def _paid_post(self, path: str, body: dict) -> dict:
+        return self._paid_request("POST", path, json_body=body)
+
+    def _paid_request(
+        self, method: str, path: str, json_body: dict | None = None
+    ) -> dict:
+        url = f"{self._base_url}{path}"
+        kwargs: dict = {}
+        if json_body is not None:
+            kwargs["json"] = json_body
+
+        max_attempts = (self._retry.max_retries + 1) if self._retry else 1
+        last_error: Exception | None = None
+
+        for attempt in range(max_attempts):
+            try:
+                logger.debug(
+                    "Request %s %s (attempt %d/%d)", method, path, attempt + 1, max_attempts
+                )
+
+                # First request — expect 402
+                resp = self._client.request(method, url, **kwargs)
+
+                if resp.status_code == 402:
+                    # x402 payment flow
+                    payment_required = self._parse_payment_required(resp)
+                    payment_payload = self._create_payment_payload(payment_required)
+                    encoded = base64.b64encode(
+                        json.dumps(payment_payload).encode()
+                    ).decode()
+
+                    # Retry with payment header
+                    headers = {"PAYMENT-SIGNATURE": encoded}
+                    resp = self._client.request(method, url, headers=headers, **kwargs)
+
+                # Check for retryable errors
+                if resp.status_code == 429:
+                    retry_after_ms = self._parse_retry_after(resp.headers)
+                    if attempt < max_attempts - 1 and self._retry:
+                        delay_ms = self._get_retry_delay_ms(attempt, retry_after_ms)
+                        logger.debug(
+                            "Rate limited (429), retrying in %dms", delay_ms
+                        )
+                        time.sleep(delay_ms / 1000)
+                        continue
+                    raise RateLimitError(
+                        f"Rate limited on {path}",
+                        endpoint=path,
+                        retry_after_ms=retry_after_ms,
+                    )
+
+                if resp.status_code >= 500 and self._should_retry(resp.status_code):
+                    if attempt < max_attempts - 1 and self._retry:
+                        delay_ms = self._get_retry_delay_ms(attempt)
+                        logger.debug(
+                            "Server error %d, retrying in %dms",
+                            resp.status_code,
+                            delay_ms,
+                        )
+                        time.sleep(delay_ms / 1000)
+                        continue
+                    # Final attempt — raise
+                    try:
+                        body_text = resp.json().get("error", resp.text)
+                    except Exception:
+                        body_text = str(resp.status_code)
+                    raise ApiError(
+                        f"Server error on {path}: {body_text}",
+                        resp.status_code,
+                        endpoint=path,
+                    )
+
+                if resp.status_code >= 400:
+                    # Non-retryable client errors
+                    try:
+                        body_text = resp.json().get("error", str(resp.status_code))
+                    except Exception:
+                        body_text = str(resp.status_code)
+                    raise ApiError(
+                        f"API error on {path}: {body_text}",
+                        resp.status_code,
+                        endpoint=path,
+                    )
+
+                return resp.json()
+
+            except (PaymentError, RateLimitError, ApiError):
+                raise
+
+            except (httpx.ConnectError, httpx.TimeoutException, httpx.ReadTimeout) as exc:
+                last_error = exc
+                if attempt < max_attempts - 1 and self._retry:
+                    delay_ms = self._get_retry_delay_ms(attempt)
+                    logger.debug(
+                        "Network error (%s), retrying in %dms",
+                        type(exc).__name__,
+                        delay_ms,
+                    )
+                    time.sleep(delay_ms / 1000)
+                    continue
+                raise NetworkError(
+                    f"Network error on {path}: {exc}",
+                    endpoint=path,
+                    cause=exc,
+                ) from exc
+
+        # Should never reach here, but just in case
+        raise NetworkError(
+            f"All {max_attempts} attempts failed for {path}",
+            endpoint=path,
+            cause=last_error,
+        )
+
+    # ─── x402 helpers ──────────────────────────────────────────
+
+    def _parse_payment_required(self, resp: httpx.Response) -> dict:
+        """Parse x402 payment requirements from 402 response."""
+        # v2: base64-encoded JSON in PAYMENT-REQUIRED header
+        header = resp.headers.get("payment-required") or resp.headers.get(
+            "PAYMENT-REQUIRED"
+        )
+        if header:
+            return json.loads(base64.b64decode(header))
+
+        # v1: JSON body
+        body = resp.json()
+        if "x402Version" in body:
+            return body
+
+        raise ValueError("Cannot parse x402 payment requirements from 402 response")
+
+    def _create_payment_payload(self, payment_required: dict) -> dict:
+        """Create EIP-3009 TransferWithAuthorization payload."""
+        # v2 format: accepts is a list of payment options
+        if "accepts" in payment_required:
+            req = payment_required["accepts"][0]
+        else:
+            req = payment_required
+
+        now = int(time.time())
+        nonce = os.urandom(32)
+
+        authorization = {
+            "from": self._account.address,
+            "to": req["payTo"],
+            "value": str(req.get("maxAmountRequired", req.get("amount", "0"))),
+            "validAfter": str(now - 600),
+            "validBefore": str(now + req.get("maxTimeoutSeconds", 300)),
+            "nonce": "0x" + nonce.hex(),
+        }
+
+        # Get token metadata for EIP-712 domain
+        extra = req.get("extra", {})
+        token_name = extra.get("name", "USD Coin")
+        token_version = extra.get("version", "2")
+        chain_id = int(req["network"].split(":")[1])
+        token_address = req["asset"]
+
+        # Sign EIP-712 typed data
+        domain = {
+            "name": token_name,
+            "version": token_version,
+            "chainId": chain_id,
+            "verifyingContract": token_address,
+        }
+
+        types = {
+            "TransferWithAuthorization": [
+                {"name": "from", "type": "address"},
+                {"name": "to", "type": "address"},
+                {"name": "value", "type": "uint256"},
+                {"name": "validAfter", "type": "uint256"},
+                {"name": "validBefore", "type": "uint256"},
+                {"name": "nonce", "type": "bytes32"},
+            ],
+        }
+
+        message = {
+            "from": authorization["from"],
+            "to": authorization["to"],
+            "value": int(authorization["value"]),
+            "validAfter": int(authorization["validAfter"]),
+            "validBefore": int(authorization["validBefore"]),
+            "nonce": bytes.fromhex(authorization["nonce"][2:]),
+        }
+
+        signed = self._account.sign_typed_data(
+            domain_data=domain,
+            message_types=types,
+            message_data=message,
+        )
+
+        return {
+            "x402Version": 2,
+            "payload": {
+                "authorization": authorization,
+                "signature": signed.signature.hex()
+                if isinstance(signed.signature, bytes)
+                else str(signed.signature),
+            },
+        }

clawfetch/errors.py

@@ -0,0 +1,49 @@
+"""ClawFetch error hierarchy — mirrors the TypeScript SDK error classes."""
+
+from __future__ import annotations
+
+
+class ClawFetchError(Exception):
+    """Base error for all ClawFetch SDK errors."""
+
+    def __init__(self, message: str, status_code: int, endpoint: str) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.endpoint = endpoint
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.status_code}, {self.endpoint!r})"
+
+
+class PaymentError(ClawFetchError):
+    """Payment-related errors (402, insufficient USDC, invalid signature)."""
+
+    def __init__(self, message: str, status_code: int = 402, endpoint: str = "") -> None:
+        super().__init__(message, status_code, endpoint)
+
+
+class NetworkError(ClawFetchError):
+    """Network errors (connection refused, DNS failure, timeout)."""
+
+    def __init__(
+        self, message: str, endpoint: str = "", cause: Exception | None = None
+    ) -> None:
+        super().__init__(message, 0, endpoint)
+        self.__cause__ = cause
+
+
+class RateLimitError(ClawFetchError):
+    """Rate limit errors (429 Too Many Requests)."""
+
+    def __init__(
+        self, message: str, endpoint: str = "", retry_after_ms: int | None = None
+    ) -> None:
+        super().__init__(message, 429, endpoint)
+        self.retry_after_ms = retry_after_ms
+
+
+class ApiError(ClawFetchError):
+    """API errors (4xx other than 402/429, 5xx)."""
+
+    def __init__(self, message: str, status_code: int, endpoint: str = "") -> None:
+        super().__init__(message, status_code, endpoint)

clawfetch-0.1.0.dist-info/METADATA

@@ -0,0 +1,268 @@
+Metadata-Version: 2.4
+Name: clawfetch
+Version: 0.1.0
+Summary: Python SDK for ClawFetch — Web Intelligence API for AI Agents (x402-native)
+License: MIT
+Project-URL: Homepage, https://api.clawfetch.ai
+Project-URL: Repository, https://github.com/clawfetch/sdk-python
+Keywords: clawfetch,x402,web-intelligence,ai-agents,crypto,usdc,base
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: eth-account>=0.13.0
+Requires-Dist: eth-abi>=5.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.15; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24.0; extra == "dev"
+
+# clawfetch
+
+Python SDK for [ClawFetch](https://api.clawfetch.ai) — Web Intelligence API for AI Agents.
+
+Pay-per-request via [x402](https://x402.org) (gasless USDC on Base). No API keys, no subscriptions.
+
+## Install
+
+```bash
+pip install clawfetch
+```
+
+## Quick Start
+
+### Sync
+
+```python
+from clawfetch import ClawFetch
+
+with ClawFetch(private_key="0x...") as cf:
+    # Fetch any URL as clean markdown ($0.001)
+    page = cf.fetch("https://example.com")
+
+    # Extract structured data ($0.003)
+    btc = cf.extract("https://coingecko.com/en/coins/bitcoin")
+    print(btc["data"])
+
+    # Multi-source research ($0.01)
+    report = cf.research("latest AI agent frameworks")
+```
+
+### Async
+
+```python
+import asyncio
+from clawfetch import AsyncClawFetch
+
+async def main():
+    async with AsyncClawFetch(private_key="0x...") as cf:
+        page = await cf.fetch("https://example.com")
+        btc = await cf.extract("https://coingecko.com/en/coins/bitcoin")
+        report = await cf.research("latest AI agent frameworks")
+
+asyncio.run(main())
+```
+
+Both clients have identical APIs — all 7 endpoints, full x402 payment flow, retry with exponential backoff, and typed error hierarchy.
+
+## API
+
+| Method | Price | Description |
+|--------|-------|-------------|
+| `fetch(url)` | $0.001 | URL → clean markdown |
+| `render(url)` | $0.002 | JS-rendered page → markdown |
+| `extract(url)` | $0.003 | Structured data from 17+ sites |
+| `research(topic)` | $0.010 | Multi-source topic research |
+| `domains_check(domains)` | $0.002 | Domain availability check |
+| `domains_suggest(query)` | $0.002 | Domain name suggestions |
+| `extractors()` | $0.001 | List available extractors |
+| `health()` | Free | API status check |
+
+## Error Handling
+
+```python
+from clawfetch import (
+    ClawFetch,
+    ClawFetchError,    # Base class
+    PaymentError,      # 402 — insufficient USDC
+    NetworkError,      # Connection/timeout failures
+    RateLimitError,    # 429 — includes retry_after_ms
+    ApiError,          # Other 4xx/5xx errors
+)
+```
+
+## Configuration
+
+```python
+from clawfetch import ClawFetch, RetryOptions
+
+cf = ClawFetch(
+    private_key="0x...",
+    base_url="https://api.clawfetch.ai",  # default
+    timeout=30.0,                          # seconds
+    retry=RetryOptions(
+        max_retries=3,
+        initial_delay_ms=500,
+        max_delay_ms=10000,
+        backoff_multiplier=2.0,
+    ),
+    debug=False,
+)
+```
+
+## Requirements
+
+- Python 3.9+
+- A wallet with USDC on Base (even $1 gives you 1,000+ requests)
+
+## License
+
+MIT
+
+---
+
+# Original README below
+
+Python SDK for [ClawFetch](https://api.clawfetch.ai) — Web Intelligence API for AI Agents.
+
+Pay-per-request via [x402](https://x402.org) (gasless USDC on Base). No API keys, no subscriptions.
+
+## Install
+
+```bash
+pip install clawfetch
+```
+
+## Quick Start
+
+```python
+from clawfetch import ClawFetch
+
+cf = ClawFetch(private_key="0x...")
+
+# Fetch any URL as clean markdown ($0.001)
+page = cf.fetch("https://example.com")
+
+# Extract structured data ($0.003)
+btc = cf.extract("https://coingecko.com/en/coins/bitcoin")
+print(btc["data"])  # {'name': 'Bitcoin', 'price': 98432.12, ...}
+
+# JS-rendered pages ($0.002)
+rendered = cf.render("https://app.uniswap.org")
+
+# Multi-source research ($0.01)
+report = cf.research("latest AI agent frameworks")
+
+# Domain availability ($0.002)
+domains = cf.domains_check(["coolstartup.com", "coolstartup.ai"])
+
+# List extractors ($0.001)
+extractors = cf.extractors()
+```
+
+## Configuration
+
+```python
+from clawfetch import ClawFetch, RetryOptions
+
+cf = ClawFetch(
+    private_key="0x...",
+    base_url="https://api.clawfetch.ai",  # default
+    timeout=30.0,                           # request timeout in seconds
+    retry=RetryOptions(                     # or False to disable retries
+        max_retries=3,
+        initial_delay_ms=500,
+        max_delay_ms=10_000,
+        backoff_multiplier=2.0,
+    ),
+    debug=False,                            # enable debug logging
+)
+```
+
+## Error Handling
+
+All errors extend `ClawFetchError` for easy catching:
+
+```python
+from clawfetch import ClawFetch, ClawFetchError, PaymentError, NetworkError, RateLimitError, ApiError
+
+cf = ClawFetch(private_key="0x...")
+
+try:
+    result = cf.fetch("https://example.com")
+except PaymentError as e:
+    print(f"Payment failed ({e.status_code}): {e}")
+except RateLimitError as e:
+    print(f"Rate limited, retry after {e.retry_after_ms}ms")
+except NetworkError as e:
+    print(f"Network error: {e}")
+except ApiError as e:
+    print(f"API error {e.status_code}: {e}")
+except ClawFetchError as e:
+    print(f"ClawFetch error: {e}")
+```
+
+| Error | Status | Retried |
+|-------|--------|---------|
+| `PaymentError` | 402 | No |
+| `RateLimitError` | 429 | Yes (with Retry-After) |
+| `ApiError` | 400, 401, 404 | No |
+| `ApiError` | 500, 502, 503, 504 | Yes |
+| `NetworkError` | Connection/timeout | Yes |
+
+## Retry Behavior
+
+By default, retries are enabled with exponential backoff + jitter:
+
+| Attempt | Delay |
+|---------|-------|
+| 1st retry | ~500ms |
+| 2nd retry | ~1,000ms |
+| 3rd retry | ~2,000ms |
+
+Respects `Retry-After` headers on 429 responses. Non-retryable errors (400, 401, 402, 404) fail immediately.
+
+Disable retries:
+
+```python
+cf = ClawFetch(private_key="0x...", retry=False)
+```
+
+## How It Works
+
+1. SDK makes a request to ClawFetch
+2. Server returns `402 Payment Required` with USDC amount
+3. SDK auto-signs an EIP-3009 gasless USDC transfer on Base
+4. Request is retried with payment header
+5. You get structured data back
+
+No gas fees. No API keys. Just USDC on Base.
+
+## Context Manager
+
+```python
+with ClawFetch(private_key="0x...") as cf:
+    data = cf.extract("https://coingecko.com/en/coins/bitcoin")
+# Client is automatically closed
+```
+
+## Requirements
+
+- Python 3.9+
+- A wallet with USDC on Base (even $1 gives you 1,000+ requests)
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+## License
+
+MIT

clawfetch-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+clawfetch/__init__.py,sha256=6AKQnl_n7a0lJyA6l5VTNCIMAVaGuu34xXFFhVGfSks,440
+clawfetch/async_client.py,sha256=3Fycu93Ih8EnimnRWv1cd80LiGNSsjf4HUeU3E9ipIU,16121
+clawfetch/client.py,sha256=lYlRdjEIM5qljnzPJx3xcB-6rjmRoOX0PgN5nzfiB3w,16726
+clawfetch/errors.py,sha256=z_lEkjD9ztWSvVe65tfn4s0YeZIx-KTTtPP26nNxTyU,1594
+clawfetch-0.1.0.dist-info/METADATA,sha256=VPXar2PTOz61Gp9fOyRaMXldJEywAfINXyPXNhtwo90,6887
+clawfetch-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+clawfetch-0.1.0.dist-info/top_level.txt,sha256=PqD6ZdHlgbhi5CypO0tAg5fLHbi3UY0U9xCtgwFFAJE,10
+clawfetch-0.1.0.dist-info/RECORD,,

clawfetch-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

clawfetch-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+clawfetch