xdive 1.5.0__py3-none-any.whl

xdive/__init__.py

@@ -0,0 +1,31 @@
+"""XDive SDK: scrape, crawl, map, search and research the web.
+
+    from xdive import XDive
+
+    xdive = XDive(api_key="ck_live_...")
+    page = xdive.scrape("https://example.com")
+    answer = xdive.research("What changed in the RBI's October 2026 policy?")
+    print(answer["answer"], answer["citations"])
+"""
+
+from .client import (
+    AsyncXDive,
+    XDive,
+    XDiveError,
+    CrawlTimeout,
+    QuotaExceeded,
+    RateLimited,
+    page_markdown,
+)
+
+__all__ = [
+    "AsyncXDive",
+    "XDive",
+    "XDiveError",
+    "CrawlTimeout",
+    "QuotaExceeded",
+    "RateLimited",
+    "page_markdown",
+]
+
+__version__ = "1.5.0"

xdive/client.py

@@ -0,0 +1,512 @@
+"""HTTP client for the XDive API (synchronous and asynchronous).
+
+Design notes:
+- One API key identifies one organization; the client discovers the
+  organization id through ``GET /whoami`` so users configure only the key.
+- ``429`` (organization concurrency/rate limit) and ``503`` with the code
+  ``SERVER_BUSY`` (node at capacity) are the API's designed answers to
+  overload and carry ``Retry-After``; the client waits as told and retries a
+  bounded number of times before raising :class:`RateLimited`. Any other
+  ``503`` (a feature that is not configured, a queue that is down) is an
+  ordinary :class:`XDiveError` and is not retried.
+- Asynchronous jobs (crawl, batch scrape) are polled with ``If-None-Match``
+  so unchanged status answers are bodiless ``304``\\s, and at the interval the
+  server suggests in ``Retry-After``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+import uuid
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Union
+
+import httpx
+
+DEFAULT_BASE_URL = "https://crawler.mehdisadeghian.com/api"
+JSON = Dict[str, Any]
+_TERMINAL = {"success", "error", "cancelled"}
+
+
+class XDiveError(Exception):
+    """An API error. ``status`` is the HTTP status; ``code`` the API's
+    machine-readable code when it sent one."""
+
+    def __init__(self, status: int, message: str, code: Optional[str] = None, retryable: bool = False):
+        super().__init__(f"{status}: {message}" if code is None else f"{status} {code}: {message}")
+        self.status = status
+        self.message = message
+        self.code = code
+        #: The server's ``retryable`` flag: the same request may succeed later.
+        self.retryable = retryable
+
+
+class RateLimited(XDiveError):
+    """Still throttled (``429``, or ``503`` with code ``SERVER_BUSY``) after
+    the configured retries."""
+
+    def __init__(self, status: int, message: str, retry_after: float, code: Optional[str] = None):
+        super().__init__(status, message, code, retryable=True)
+        self.retry_after = retry_after
+
+
+class QuotaExceeded(XDiveError):
+    """The organization's monthly quota is used up (HTTP 402)."""
+
+
+class CrawlTimeout(XDiveError):
+    """A crawl did not finish within ``timeout`` seconds."""
+
+    def __init__(self, crawl_id: str, timeout: float):
+        super().__init__(408, f"crawl {crawl_id} did not finish within {timeout:.0f}s")
+        self.crawl_id = crawl_id
+
+
+def _error_from(response: httpx.Response) -> XDiveError:
+    code = None
+    retryable = False
+    message = response.text.strip()[:500] or response.reason_phrase
+    try:
+        payload = response.json()
+        if isinstance(payload, dict):
+            message = payload.get("error") or payload.get("message") or message
+            code = payload.get("code")
+            retryable = payload.get("retryable") is True
+    except ValueError:
+        pass
+    if response.status_code == 402:
+        return QuotaExceeded(response.status_code, message, code, retryable)
+    return XDiveError(response.status_code, message, code, retryable)
+
+
+def _is_overload(response: httpx.Response) -> bool:
+    """Whether a response is the API's "slow down" answer, which is retried
+    after ``Retry-After``: any ``429``, and a ``503`` whose JSON body carries
+    the code ``SERVER_BUSY`` (the node is at capacity). Other ``503``s, such
+    as a feature that is not configured, are ordinary errors."""
+    if response.status_code == 429:
+        return True
+    if response.status_code not in (409, 503):
+        return False
+    try:
+        payload = response.json()
+    except ValueError:
+        return False
+    if not isinstance(payload, dict):
+        return False
+    # 409 IDEMPOTENCY_KEY_IN_PROGRESS: the first attempt of this POST is still
+    # running; the retry receives its stored response once it finishes.
+    return payload.get("code") == ("SERVER_BUSY" if response.status_code == 503 else "IDEMPOTENCY_KEY_IN_PROGRESS")
+
+
+def _attempt_headers(method: str, headers: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]:
+    """Headers for every attempt of one request. A POST gets one
+    Idempotency-Key for all its attempts, so a retry after a timeout or a
+    dropped connection gets the first response back instead of running (and
+    billing) the work again."""
+    if method != "POST":
+        return headers
+    return {"Idempotency-Key": str(uuid.uuid4()), **(headers or {})}
+
+
+def _backoff(attempt: int) -> float:
+    return min(2.0 ** attempt, 8.0)
+
+
+def page_markdown(page: Mapping[str, Any]) -> str:
+    """The Markdown of a page, whichever field carries it:
+    ``artifacts.markdown.content`` (a scrape, or a crawl with ``formats``) or
+    ``markdown`` (a crawl without ``formats``). Empty when there is none."""
+    artifacts = page.get("artifacts") or {}
+    return (artifacts.get("markdown") or {}).get("content") or page.get("markdown") or ""
+
+
+def _retry_after(response: httpx.Response, default: float) -> float:
+    try:
+        return max(0.0, float(response.headers.get("Retry-After", default)))
+    except ValueError:
+        return default
+
+
+def _drop_none(payload: JSON) -> JSON:
+    return {key: value for key, value in payload.items() if value is not None}
+
+
+class _Base:
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        org_id: Optional[str] = None,
+        timeout: float = 120.0,
+        max_retries: int = 3,
+    ):
+        self.api_key = api_key or os.environ.get("XDIVE_API_KEY") or os.environ.get("CRAWLER_API_KEY") or ""
+        if not self.api_key:
+            raise ValueError("an API key is required (argument api_key or XDIVE_API_KEY)")
+        self.base_url = (base_url or os.environ.get("XDIVE_BASE_URL") or os.environ.get("CRAWLER_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
+        self._org_id = org_id
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self._headers = {
+            "X-API-Key": self.api_key,
+            "Accept": "application/json",
+            "User-Agent": "xdive-python/1.5.0",
+        }
+
+    # --- payload builders (shared by sync and async) -----------------------
+
+    @staticmethod
+    def _scrape_payload(url: str, formats: Optional[Sequence[Union[str, JSON]]], options: JSON) -> JSON:
+        payload: JSON = {"url": url}
+        if formats:
+            payload["formats"] = [f if isinstance(f, dict) else {"type": f} for f in formats]
+        payload.update(_drop_none(options))
+        return payload
+
+    @staticmethod
+    def _crawl_payload(url: str, options: JSON) -> JSON:
+        payload: JSON = {"url": url}
+        formats = options.pop("formats", None)
+        if formats:
+            payload["formats"] = [f if isinstance(f, dict) else {"type": f} for f in formats]
+        payload.update(_drop_none(options))
+        return payload
+
+    @staticmethod
+    def _batch_payload(urls: Iterable[str], options: JSON) -> JSON:
+        payload: JSON = {"urls": list(urls)}
+        formats = options.pop("formats", None)
+        if formats:
+            payload["formats"] = [f if isinstance(f, dict) else {"type": f} for f in formats]
+        payload.update(_drop_none(options))
+        return payload
+
+    @staticmethod
+    def _llms_txt_payload(url: str, max_urls: Optional[int], full_text: bool) -> JSON:
+        return _drop_none({"url": url, "max_urls": max_urls, "full_text": full_text or None})
+
+
+class XDive(_Base):
+    """Synchronous client.
+
+    Args:
+        api_key: an organization API key (``ck_live_…``); defaults to
+            ``XDIVE_API_KEY`` (or the older ``CRAWLER_API_KEY``).
+        base_url: API origin including ``/api`` for a self-hosted instance;
+            defaults to ``XDIVE_BASE_URL`` (or ``CRAWLER_BASE_URL``) or the hosted service.
+        org_id: skip discovery when you already know the organization id.
+        timeout: per-request timeout in seconds.
+        max_retries: retries on ``429`` and on ``503`` with code
+            ``SERVER_BUSY`` (each waits ``Retry-After``).
+    """
+
+    def __init__(self, api_key: Optional[str] = None, **kwargs: Any):
+        super().__init__(api_key, **kwargs)
+        self._http = httpx.Client(base_url=self.base_url, headers=self._headers, timeout=self.timeout)
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "XDive":
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    # --- transport ---------------------------------------------------------
+
+    def _request(self, method: str, path: str, *, json: Optional[JSON] = None, params: Optional[JSON] = None,
+                 headers: Optional[Dict[str, str]] = None) -> httpx.Response:
+        attempt = 0
+        headers = _attempt_headers(method, headers)
+        while True:
+            try:
+                response = self._http.request(method, path, json=json, params=params, headers=headers)
+            except httpx.TransportError:
+                # Timeouts and network failures: safe to retry GETs, and POSTs
+                # thanks to the Idempotency-Key.
+                if method in ("GET", "POST") and attempt < self.max_retries:
+                    time.sleep(_backoff(attempt))
+                    attempt += 1
+                    continue
+                raise
+            if _is_overload(response):
+                if attempt < self.max_retries:
+                    attempt += 1
+                    time.sleep(_retry_after(response, 1.0))
+                    continue
+                error = _error_from(response)
+                raise RateLimited(response.status_code, error.message, _retry_after(response, 1.0), error.code)
+            if response.status_code >= 400:
+                raise _error_from(response)
+            return response
+
+    def _post(self, path: str, payload: JSON) -> JSON:
+        return self._request("POST", path, json=payload).json()
+
+    @property
+    def org_id(self) -> str:
+        if self._org_id is None:
+            self._org_id = self._request("GET", "/whoami").json()["orgId"]
+        return self._org_id
+
+    def _org(self, suffix: str) -> str:
+        return f"/orgs/{self.org_id}{suffix}"
+
+    # --- identity and usage ------------------------------------------------
+
+    def whoami(self) -> JSON:
+        """Who the API key is: organization, scopes, plan."""
+        return self._request("GET", "/whoami").json()
+
+    # --- pages ---------------------------------------------------------------
+
+    def scrape(self, url: str, formats: Optional[Sequence[Union[str, JSON]]] = None, **options: Any) -> JSON:
+        """Scrape one page synchronously.
+
+        ``formats`` are ``markdown`` (default), ``text``, ``html``,
+        ``rawHtml``, ``links``, ``images``, ``json``, ``summary``,
+        ``question``, ``highlights``, ``branding``, ``audio``, ``screenshot``,
+        ``product`` and ``menu``; pass a dict such as
+        ``{"type": "json", "schema": {...}}`` to set a format's options.
+        Other keyword options map to the
+        API fields (``max_age``, ``headers``, ``respect_robots``,
+        ``redact_pii``, ``actions``, ``wait_for``…). Returns the ``data``
+        page object.
+        """
+        return self._post(self._org("/scrape"), self._scrape_payload(url, formats, options))["data"]
+
+    def start_crawl(self, url: str, **options: Any) -> str:
+        """Start a crawl and return its id (see :meth:`crawl` to wait)."""
+        return self._post(self._org("/crawl"), self._crawl_payload(url, options))["id"]
+
+    def get_crawl(self, crawl_id: str) -> JSON:
+        """Current status and, once finished, results of a crawl."""
+        return self._request("GET", self._org(f"/crawls/{crawl_id}")).json()
+
+    def cancel_crawl(self, crawl_id: str) -> None:
+        """Cancel a queued or running crawl (or batch scrape)."""
+        self._request("DELETE", self._org(f"/crawls/{crawl_id}"))
+
+    def crawl_errors(self, crawl_id: str) -> JSON:
+        """The pages of a crawl that failed: ``errors`` (``url``/``error``
+        pairs) and ``robots_blocked`` (URLs refused by robots.txt)."""
+        return self._request("GET", self._org(f"/crawls/{crawl_id}/errors")).json()
+
+    def wait_for_crawl(self, crawl_id: str, *, timeout: float = 600.0, poll_interval: float = 2.0) -> JSON:
+        """Poll until the crawl finishes, honouring the server's ETag and
+        Retry-After, and return the final record."""
+        deadline = time.monotonic() + timeout
+        etag: Optional[str] = None
+        last: Optional[JSON] = None
+        while True:
+            headers = {"If-None-Match": etag} if etag else None
+            response = self._request("GET", self._org(f"/crawls/{crawl_id}"), headers=headers)
+            if response.status_code != 304:
+                last = response.json()
+                etag = response.headers.get("ETag")
+                if last.get("status") in _TERMINAL:
+                    return last
+            if time.monotonic() >= deadline:
+                raise CrawlTimeout(crawl_id, timeout)
+            time.sleep(min(_retry_after(response, poll_interval), max(0.0, deadline - time.monotonic())))
+
+    def crawl(self, url: str, *, timeout: float = 600.0, poll_interval: float = 2.0, **options: Any) -> JSON:
+        """Crawl a site and wait for the result. Options: ``max_depth``,
+        ``max_pages``, ``same_domain``, ``formats``, ``include_paths``,
+        ``exclude_paths``, ``render``, ``adaptive_fetch``, ``headers``,
+        ``max_age``, ``webhook_url``, ``llm_extract``, ``extract_schema``…"""
+        return self.wait_for_crawl(self.start_crawl(url, **options), timeout=timeout, poll_interval=poll_interval)
+
+    def batch_scrape(self, urls: Sequence[str], *, wait: bool = True, timeout: float = 600.0, **options: Any) -> Union[str, JSON]:
+        """Scrape up to 100 URLs through the job pipeline; returns the crawl
+        record (or the job id when ``wait=False``)."""
+        crawl_id = self._post(self._org("/batch/scrape"), self._batch_payload(urls, options))["id"]
+        return self.wait_for_crawl(crawl_id, timeout=timeout) if wait else crawl_id
+
+    def map(self, url: str, *, search: Optional[str] = None, limit: Optional[int] = None,
+            max_depth: Optional[int] = None) -> List[str]:
+        """Discover a site's URLs (sitemap + links), optionally filtered."""
+        payload = _drop_none({"url": url, "search": search, "limit": limit, "max_depth": max_depth})
+        return self._post(self._org("/map"), payload)["urls"]
+
+    # --- discovery -----------------------------------------------------------
+
+    def search(self, query: str, **options: Any) -> JSON:
+        """Web search. Options: ``sources`` (web/news/images), ``limit``,
+        ``country``, ``freshness``, ``include_domains``, ``exclude_domains``,
+        ``search_depth`` ("advanced" also reads the top results and attaches
+        each one's query-relevant ``passages`` and a ``relevance_score``;
+        results keep the engine's order), ``auto_parameters``,
+        ``include_answer``, ``scrape``."""
+        payload: JSON = {"query": query}
+        payload.update(_drop_none(options))
+        return self._post(self._org("/search"), payload)
+
+    def research(self, query: str, *, depth: str = "quick", output: Optional[str] = None,
+                 schema: Optional[JSON] = None, **options: Any) -> JSON:
+        """Answer a question from the web with verified citations.
+
+        ``depth`` is ``quick`` (seconds) or ``deep`` (1–2 minutes, planned
+        sub-queries, report). Pass ``schema`` for structured ``data`` instead
+        of prose, ``no_cache=True`` to bypass the short-lived result cache.
+        Returns ``answer``/``data``, ``citations``, ``sources``,
+        ``confidence`` and ``usage`` (``cached`` is true for a cache hit).
+        ``usage.plan``, when present, says how the question was planned:
+        ``single``, ``multi_hop`` (answered through an intermediate search)
+        or ``unavailable`` (the planning step was too slow or failed)."""
+        payload: JSON = {"query": query, "depth": depth}
+        payload.update(_drop_none({"output": output, "schema": schema}))
+        payload.update(_drop_none(options))
+        return self._post(self._org("/research"), payload)
+
+    def extract(self, urls: Sequence[str], *, prompt: Optional[str] = None, schema: Optional[JSON] = None,
+                **options: Any) -> JSON:
+        """Extract structured data across up to 10 URLs with a prompt and/or
+        a JSON Schema."""
+        payload: JSON = {"urls": list(urls)}
+        payload.update(_drop_none({"prompt": prompt, "schema": schema}))
+        payload.update(_drop_none(options))
+        return self._post(self._org("/extract"), payload)
+
+    def llms_txt(self, url: str, *, max_urls: Optional[int] = None, full_text: bool = False) -> JSON:
+        """Generate llms.txt (and optionally llms-full.txt) for a site."""
+        return self._post(self._org("/llmstxt"), self._llms_txt_payload(url, max_urls, full_text))
+
+
+class AsyncXDive(_Base):
+    """Asynchronous client with the same methods as :class:`XDive`."""
+
+    def __init__(self, api_key: Optional[str] = None, **kwargs: Any):
+        super().__init__(api_key, **kwargs)
+        self._http = httpx.AsyncClient(base_url=self.base_url, headers=self._headers, timeout=self.timeout)
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncXDive":
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.aclose()
+
+    async def _request(self, method: str, path: str, *, json: Optional[JSON] = None, params: Optional[JSON] = None,
+                       headers: Optional[Dict[str, str]] = None) -> httpx.Response:
+        attempt = 0
+        headers = _attempt_headers(method, headers)
+        while True:
+            try:
+                response = await self._http.request(method, path, json=json, params=params, headers=headers)
+            except httpx.TransportError:
+                if method in ("GET", "POST") and attempt < self.max_retries:
+                    await asyncio.sleep(_backoff(attempt))
+                    attempt += 1
+                    continue
+                raise
+            if _is_overload(response):
+                if attempt < self.max_retries:
+                    attempt += 1
+                    await asyncio.sleep(_retry_after(response, 1.0))
+                    continue
+                error = _error_from(response)
+                raise RateLimited(response.status_code, error.message, _retry_after(response, 1.0), error.code)
+            if response.status_code >= 400:
+                raise _error_from(response)
+            return response
+
+    async def _post(self, path: str, payload: JSON) -> JSON:
+        return (await self._request("POST", path, json=payload)).json()
+
+    async def org_id(self) -> str:
+        if self._org_id is None:
+            self._org_id = (await self._request("GET", "/whoami")).json()["orgId"]
+        return self._org_id
+
+    async def _org(self, suffix: str) -> str:
+        return f"/orgs/{await self.org_id()}{suffix}"
+
+    async def whoami(self) -> JSON:
+        return (await self._request("GET", "/whoami")).json()
+
+    async def scrape(self, url: str, formats: Optional[Sequence[Union[str, JSON]]] = None, **options: Any) -> JSON:
+        return (await self._post(await self._org("/scrape"), self._scrape_payload(url, formats, options)))["data"]
+
+    async def start_crawl(self, url: str, **options: Any) -> str:
+        return (await self._post(await self._org("/crawl"), self._crawl_payload(url, options)))["id"]
+
+    async def get_crawl(self, crawl_id: str) -> JSON:
+        return (await self._request("GET", await self._org(f"/crawls/{crawl_id}"))).json()
+
+    async def cancel_crawl(self, crawl_id: str) -> None:
+        """Cancel a queued or running crawl (or batch scrape)."""
+        await self._request("DELETE", await self._org(f"/crawls/{crawl_id}"))
+
+    async def crawl_errors(self, crawl_id: str) -> JSON:
+        """The pages of a crawl that failed: ``errors`` (``url``/``error``
+        pairs) and ``robots_blocked`` (URLs refused by robots.txt)."""
+        return (await self._request("GET", await self._org(f"/crawls/{crawl_id}/errors"))).json()
+
+    async def wait_for_crawl(self, crawl_id: str, *, timeout: float = 600.0, poll_interval: float = 2.0) -> JSON:
+        """Poll until the crawl finishes, honouring the server's ETag and
+        Retry-After, and return the final record."""
+        deadline = time.monotonic() + timeout
+        etag: Optional[str] = None
+        while True:
+            headers = {"If-None-Match": etag} if etag else None
+            response = await self._request("GET", await self._org(f"/crawls/{crawl_id}"), headers=headers)
+            if response.status_code != 304:
+                last = response.json()
+                etag = response.headers.get("ETag")
+                if last.get("status") in _TERMINAL:
+                    return last
+            if time.monotonic() >= deadline:
+                raise CrawlTimeout(crawl_id, timeout)
+            await asyncio.sleep(min(_retry_after(response, poll_interval), max(0.0, deadline - time.monotonic())))
+
+    async def crawl(self, url: str, *, timeout: float = 600.0, poll_interval: float = 2.0, **options: Any) -> JSON:
+        """Crawl a site and wait for the result (see :meth:`XDive.crawl`)."""
+        return await self.wait_for_crawl(await self.start_crawl(url, **options), timeout=timeout, poll_interval=poll_interval)
+
+    async def batch_scrape(self, urls: Sequence[str], *, wait: bool = True, timeout: float = 600.0,
+                           **options: Any) -> Union[str, JSON]:
+        """Scrape up to 100 URLs through the job pipeline; returns the crawl
+        record (or the job id when ``wait=False``)."""
+        crawl_id = (await self._post(await self._org("/batch/scrape"), self._batch_payload(urls, options)))["id"]
+        return await self.wait_for_crawl(crawl_id, timeout=timeout) if wait else crawl_id
+
+    async def map(self, url: str, *, search: Optional[str] = None, limit: Optional[int] = None,
+                  max_depth: Optional[int] = None) -> List[str]:
+        """Discover a site's URLs (sitemap + links), optionally filtered."""
+        payload = _drop_none({"url": url, "search": search, "limit": limit, "max_depth": max_depth})
+        return (await self._post(await self._org("/map"), payload))["urls"]
+
+    async def search(self, query: str, **options: Any) -> JSON:
+        """Web search; same options as :meth:`XDive.search` (``search_depth``
+        "advanced" attaches query-relevant ``passages`` and a
+        ``relevance_score`` to the top results, in the engine's order)."""
+        payload: JSON = {"query": query}
+        payload.update(_drop_none(options))
+        return await self._post(await self._org("/search"), payload)
+
+    async def research(self, query: str, *, depth: str = "quick", output: Optional[str] = None,
+                       schema: Optional[JSON] = None, **options: Any) -> JSON:
+        payload: JSON = {"query": query, "depth": depth}
+        payload.update(_drop_none({"output": output, "schema": schema}))
+        payload.update(_drop_none(options))
+        return await self._post(await self._org("/research"), payload)
+
+    async def extract(self, urls: Iterable[str], *, prompt: Optional[str] = None, schema: Optional[JSON] = None,
+                      **options: Any) -> JSON:
+        """Extract structured data across up to 10 URLs with a prompt and/or
+        a JSON Schema."""
+        payload: JSON = {"urls": list(urls)}
+        payload.update(_drop_none({"prompt": prompt, "schema": schema}))
+        payload.update(_drop_none(options))
+        return await self._post(await self._org("/extract"), payload)
+
+    async def llms_txt(self, url: str, *, max_urls: Optional[int] = None, full_text: bool = False) -> JSON:
+        """Generate llms.txt (and optionally llms-full.txt) for a site."""
+        return await self._post(await self._org("/llmstxt"), self._llms_txt_payload(url, max_urls, full_text))

xdive/langchain.py

@@ -0,0 +1,149 @@
+"""LangChain integration: tools for agents and a document loader.
+
+    from xdive.langchain import XDiveSearchTool, XDiveResearchTool, XDiveLoader
+
+    tools = [XDiveSearchTool(), XDiveScrapeTool(), XDiveResearchTool()]
+    docs = XDiveLoader("https://docs.example.com", mode="crawl", max_pages=50).load()
+
+Requires ``langchain-core`` (``pip install xdive[langchain]``).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Iterator, List, Optional, Type
+
+try:
+    from langchain_core.documents import Document
+    from langchain_core.document_loaders import BaseLoader
+    from langchain_core.tools import BaseTool
+    from pydantic import BaseModel, Field, PrivateAttr
+except ImportError as exc:  # pragma: no cover - import guard
+    raise ImportError("xdive.langchain needs langchain-core: pip install 'xdive[langchain]'") from exc
+
+from .client import XDive, page_markdown
+
+
+def _compact(obj: Any) -> str:
+    return json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
+
+
+class _XDiveTool(BaseTool):
+    """Base: shares one client across tools."""
+
+    _client: XDive = PrivateAttr()
+
+    def __init__(self, client: Optional[XDive] = None, **kwargs: Any):
+        super().__init__(**kwargs)
+        self._client = client or XDive()
+
+
+class SearchInput(BaseModel):
+    query: str = Field(description="What to search the web for")
+    limit: int = Field(default=5, description="How many results (1-20)")
+    freshness: Optional[str] = Field(default=None, description="pd, pw, pm or py to prefer recent pages")
+
+
+class XDiveSearchTool(_XDiveTool):
+    """Web search that returns, for each result, the passages most relevant
+    to the query — not whole pages."""
+
+    name: str = "xdive_search"
+    description: str = (
+        "Search the web. Returns the top results with their most relevant passages "
+        "and a relevance score. Use it to find current information or sources; "
+        "use xdive_research when you need a synthesized answer with citations."
+    )
+    args_schema: Type[BaseModel] = SearchInput
+
+    def _run(self, query: str, limit: int = 5, freshness: Optional[str] = None) -> str:
+        response = self._client.search(query, limit=max(1, min(limit, 20)), freshness=freshness,
+                                       search_depth="advanced", auto_parameters=True)
+        results = [
+            {
+                "title": item.get("title"),
+                "url": item.get("url"),
+                "snippet": item.get("description"),
+                "passages": [p["text"] for p in item.get("passages", [])],
+                "relevance": item.get("relevance_score"),
+            }
+            for item in response.get("results", [])
+        ]
+        return _compact(results)
+
+
+class ScrapeInput(BaseModel):
+    url: str = Field(description="The http(s) URL to read")
+
+
+class XDiveScrapeTool(_XDiveTool):
+    """Read one page as Markdown (JavaScript rendered and bot filters handled
+    automatically)."""
+
+    name: str = "xdive_scrape"
+    description: str = "Fetch a web page and return its main content as Markdown."
+    args_schema: Type[BaseModel] = ScrapeInput
+
+    def _run(self, url: str) -> str:
+        return page_markdown(self._client.scrape(url, formats=["markdown"]))[:60_000]
+
+
+class ResearchInput(BaseModel):
+    query: str = Field(description="The question to answer from the web")
+    depth: str = Field(default="quick", description="quick (seconds) or deep (minutes, more sources)")
+
+
+class XDiveResearchTool(_XDiveTool):
+    """Answer a question from several web sources with verified citations."""
+
+    name: str = "xdive_research"
+    description: str = (
+        "Research a question on the web and return an answer in which every claim "
+        "cites a source, plus the list of citations. Use for questions needing "
+        "several sources or up-to-date facts."
+    )
+    args_schema: Type[BaseModel] = ResearchInput
+
+    def _run(self, query: str, depth: str = "quick") -> str:
+        result = self._client.research(query, depth=depth)
+        return _compact({
+            "answer": result.get("answer") or result.get("data"),
+            "confidence": result.get("confidence"),
+            "citations": [{"n": c["number"], "url": c["url"], "title": c.get("title")} for c in result.get("citations", [])],
+        })
+
+
+class XDiveLoader(BaseLoader):
+    """Load pages as LangChain documents.
+
+    ``mode="scrape"`` loads the one URL; ``mode="crawl"`` crawls from it
+    (``max_pages``, ``max_depth`` and other crawl options apply).
+    """
+
+    def __init__(self, url: str, *, mode: str = "scrape", client: Optional[XDive] = None, **options: Any):
+        if mode not in ("scrape", "crawl"):
+            raise ValueError("mode must be 'scrape' or 'crawl'")
+        self.url = url
+        self.mode = mode
+        self.options = options
+        self._client = client or XDive()
+
+    def lazy_load(self) -> Iterator[Document]:
+        if self.mode == "scrape":
+            yield self._document(self._client.scrape(self.url, formats=["markdown"], **self.options))
+            return
+        record = self._client.crawl(self.url, formats=["markdown"], **self.options)
+        for page in record.get("results") or []:
+            if page.get("error"):
+                continue
+            yield self._document(page)
+
+    def load(self) -> List[Document]:
+        return list(self.lazy_load())
+
+    @staticmethod
+    def _document(page: dict) -> Document:
+        content = page_markdown(page)
+        metadata = {key: page.get(key) for key in ("url", "title", "description", "lang", "fetched_at") if page.get(key)}
+        metadata["source"] = page.get("url")
+        return Document(page_content=content, metadata=metadata)

xdive/llama_index.py

@@ -0,0 +1,38 @@
+"""LlamaIndex integration (``pip install xdive[llama-index]``).
+
+    from xdive.llama_index import XDiveReader
+
+    documents = XDiveReader().load_data("https://docs.example.com", mode="crawl", max_pages=50)
+"""
+
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+try:
+    from llama_index.core.readers.base import BaseReader
+    from llama_index.core.schema import Document
+except ImportError as exc:  # pragma: no cover - import guard
+    raise ImportError("xdive.llama_index needs llama-index-core: pip install 'xdive[llama-index]'") from exc
+
+from .client import XDive, page_markdown
+
+
+class XDiveReader(BaseReader):
+    """Reads one page (``mode="scrape"``) or a whole site (``mode="crawl"``)
+    into LlamaIndex documents."""
+
+    def __init__(self, client: Optional[XDive] = None):
+        self._client = client or XDive()
+
+    def load_data(self, url: str, mode: str = "scrape", **options: Any) -> List[Document]:
+        if mode == "scrape":
+            pages = [self._client.scrape(url, formats=["markdown"], **options)]
+        elif mode == "crawl":
+            pages = [p for p in self._client.crawl(url, formats=["markdown"], **options).get("results") or [] if not p.get("error")]
+        else:
+            raise ValueError("mode must be 'scrape' or 'crawl'")
+        documents = []
+        for page in pages:
+            documents.append(Document(text=page_markdown(page), metadata={"url": page.get("url"), "title": page.get("title")}))
+        return documents

xdive-1.5.0.dist-info/METADATA

@@ -0,0 +1,185 @@
+Metadata-Version: 2.5
+Name: xdive
+Version: 1.5.0
+Summary: Python SDK for XDive: scrape, crawl, map, search and research the web with citations, for AI agents.
+Project-URL: Homepage, https://crawler.mehdisadeghian.com
+Project-URL: Documentation, https://crawler.mehdisadeghian.com/docs/sdk-python
+Author: XDive
+License: MIT
+License-File: LICENSE
+Keywords: agents,crawler,langchain,llm,markdown,mcp,rag,research,search,web scraping,xdive
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: httpx<1,>=0.25
+Provides-Extra: dev
+Requires-Dist: langchain-core>=0.2; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2; extra == 'langchain'
+Provides-Extra: llama-index
+Requires-Dist: llama-index-core>=0.10; extra == 'llama-index'
+Description-Content-Type: text/markdown
+
+# xdive
+
+Python client for [XDive](https://crawler.mehdisadeghian.com): scrape pages
+to Markdown, crawl and map sites, search the web, and answer questions with
+verified citations. Synchronous and asynchronous clients, typed errors,
+automatic retries, and LangChain and LlamaIndex integrations.
+
+Full documentation: https://crawler.mehdisadeghian.com/docs/sdk-python
+
+## Install
+
+```bash
+pip install xdive
+pip install "xdive[langchain]"     # LangChain tools and document loader
+pip install "xdive[llama-index]"   # LlamaIndex reader
+```
+
+Requires Python 3.9 or later. The only runtime dependency is `httpx`.
+
+## Quickstart
+
+```python
+from xdive import XDive, page_markdown
+
+with XDive() as xdive:  # reads XDIVE_API_KEY
+    page = xdive.scrape("https://example.com", formats=["markdown", "links"])
+    print(page_markdown(page))
+    site = xdive.crawl("https://docs.example.com", max_pages=50)  # waits for the crawl
+    answer = xdive.research("What does PEP 639 change about license metadata?")
+    print(answer["answer"])
+    for citation in answer["citations"]:
+        print(citation["number"], citation["url"])
+```
+
+Create an API key under **API Keys** in the dashboard. A key belongs to one
+organization; the client looks the organization up once through
+`GET /whoami`, so the key is all you configure.
+
+## Configuration
+
+| Argument | Environment variable | Default | Description |
+|---|---|---|---|
+| `api_key` | `XDIVE_API_KEY` (or `CRAWLER_API_KEY`) | (required) | Organization API key (`ck_live_…`). |
+| `base_url` | `XDIVE_BASE_URL` (or `CRAWLER_BASE_URL`) | `https://crawler.mehdisadeghian.com/api` | API base URL, including `/api`. |
+| `org_id` | | discovered | Skip the `/whoami` lookup when you know the organization id. |
+| `timeout` | | `120.0` | Per-request timeout in seconds. |
+| `max_retries` | | `3` | Retries of overload responses: `429`, and `503` with code `SERVER_BUSY`. |
+
+A missing key raises `ValueError` when the client is created.
+
+## Methods
+
+`XDive` and `AsyncXDive` have the same methods; on `AsyncXDive` each is
+a coroutine. Keyword options are passed through as the API's JSON fields.
+
+| Method | API call | Returns |
+|---|---|---|
+| `whoami()` | `GET /whoami` | The key's organization, scopes and plan. |
+| `scrape(url, formats=None, **options)` | `POST /orgs/{id}/scrape` | The page object (`data`). |
+| `crawl(url, *, timeout=600, poll_interval=2, **options)` | `POST /orgs/{id}/crawl`, then polls | The finished crawl record with `results`. |
+| `start_crawl(url, **options)` | `POST /orgs/{id}/crawl` | The crawl id. |
+| `get_crawl(crawl_id)` | `GET /orgs/{id}/crawls/{crawlId}` | The crawl record. |
+| `wait_for_crawl(crawl_id, *, timeout=600, poll_interval=2)` | polls `GET /orgs/{id}/crawls/{crawlId}` | The finished crawl record. |
+| `cancel_crawl(crawl_id)` | `DELETE /orgs/{id}/crawls/{crawlId}` | `None`. |
+| `crawl_errors(crawl_id)` | `GET /orgs/{id}/crawls/{crawlId}/errors` | Failed pages (`errors`) and `robots_blocked` URLs. |
+| `batch_scrape(urls, *, wait=True, timeout=600, **options)` | `POST /orgs/{id}/batch/scrape` | The finished record, or the job id with `wait=False`. |
+| `map(url, *, search=None, limit=None, max_depth=None)` | `POST /orgs/{id}/map` | A list of URLs. |
+| `search(query, **options)` | `POST /orgs/{id}/search` | The search response with `results`. |
+| `research(query, *, depth="quick", output=None, schema=None, **options)` | `POST /orgs/{id}/research` | `answer` or `data`, `citations`, `sources`, `confidence`, `usage`. |
+| `extract(urls, *, prompt=None, schema=None, **options)` | `POST /orgs/{id}/extract` | The extracted data. |
+| `llms_txt(url, *, max_urls=None, full_text=False)` | `POST /orgs/{id}/llmstxt` | `llms_txt` (and `llms_full_txt`). |
+
+`XDive.org_id` is a property; on `AsyncXDive` it is a coroutine,
+`await client.org_id()`.
+
+`formats` takes the API's format names (`markdown`, `text`, `html`,
+`rawHtml`, `links`, `images`, `json`, `summary`, `question`, `highlights`,
+`branding`, `audio`, `screenshot`, `product`, `menu`) or dicts such as
+`{"type": "json", "schema": {...}}`. `page_markdown(page)` returns a page's
+Markdown whichever field carries it (`artifacts.markdown.content` or
+`markdown`).
+
+## Async
+
+```python
+import asyncio
+from xdive import AsyncXDive
+
+async def main():
+    async with AsyncXDive() as client:
+        pages = await asyncio.gather(*(client.scrape(u) for u in ["https://a.example", "https://b.example"]))
+
+asyncio.run(main())
+```
+
+Close clients you do not use as context managers: `xdive.close()` or
+`await client.aclose()`.
+
+## Errors and retries
+
+| Exception | When | Attributes |
+|---|---|---|
+| `XDiveError` | Any API error (status 400 and above). Base class of the others. | `status`, `message`, `code` |
+| `RateLimited` | Still overloaded (`429`, or `503` with code `SERVER_BUSY`) after `max_retries` retries. | `retry_after` (seconds) |
+| `QuotaExceeded` | `402`: the organization's monthly quota is used up. | |
+| `CrawlTimeout` | `crawl`, `wait_for_crawl` or `batch_scrape` waited longer than `timeout`. The job keeps running on the server. | `crawl_id` |
+
+Only overload is retried: a `429` (organization rate or concurrency limit)
+and a `503` whose JSON body has the code `SERVER_BUSY` (server at capacity).
+The client waits the number of seconds in the response's `Retry-After`
+header (1 second when absent) before each retry. Every other error, including
+any other `503` (for example a feature the server has not configured), raises
+`XDiveError` at once with the API's `status`, `message` and `code`. Network
+errors and timeouts surface as `httpx` exceptions.
+
+Crawl polling sends `If-None-Match`, so an unchanged status comes back as an
+empty `304`, and waits the interval the server sets in `Retry-After`,
+falling back to `poll_interval`.
+
+## Integrations
+
+```python
+from xdive.langchain import XDiveLoader, XDiveResearchTool, XDiveScrapeTool, XDiveSearchTool
+
+tools = [XDiveSearchTool(), XDiveScrapeTool(), XDiveResearchTool()]
+docs = XDiveLoader("https://docs.example.com", mode="crawl", max_pages=100).load()
+```
+
+```python
+from xdive.llama_index import XDiveReader
+
+documents = XDiveReader().load_data("https://docs.example.com", mode="crawl", max_pages=100)
+```
+
+`XDiveSearchTool` returns each result's query-relevant passages rather than
+whole pages, which keeps an agent's context small. See the
+[LangChain](https://crawler.mehdisadeghian.com/docs/langchain) and
+[LlamaIndex](https://crawler.mehdisadeghian.com/docs/llamaindex) guides.
+
+## Self-hosted
+
+```python
+xdive = XDive(api_key="ck_live_…", base_url="https://your-host/api")
+```
+
+or set `XDIVE_BASE_URL=https://your-host/api`.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT. The license text is in the `LICENSE` file included in the package.

xdive-1.5.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+xdive/__init__.py,sha256=cmN08KYeXc8950h2wxdFOjWJ4KjVdk5TBupFmuAmp3o,621
+xdive/client.py,sha256=zpF7Va6ojjYKvMeplQ46U9M4Vq-M7SoNG8WZMOhUKPE,24086
+xdive/langchain.py,sha256=I9Lg4pTbG7v5BUV98F_uI9iq9TlNJLIgKnIfhADXCiM,5712
+xdive/llama_index.py,sha256=snmUB2wJaSRMmdSJhcmYCnuiiB4HAWttF05LlRP54tM,1501
+xdive/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xdive-1.5.0.dist-info/METADATA,sha256=1QwOfZ-bUVpYm-XK9R_xVuOAbbgSF8OPBYkp_u-urms,7907
+xdive-1.5.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+xdive-1.5.0.dist-info/licenses/LICENSE,sha256=jiN0WDt_r4yK0PaGOkjmEFbdvfpXEH1f2Eyp875GHEY,1072
+xdive-1.5.0.dist-info/RECORD,,

xdive-1.5.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

xdive-1.5.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mehdi Sadeghian
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.