datafuel 0.5.0__py3-none-any.whl

datafuel/_client.py

@@ -0,0 +1,1327 @@
+"""The sync and async clients. They only move bytes; :mod:`datafuel._core` decides."""
+
+from __future__ import annotations
+
+import asyncio
+import math
+import os
+import time
+from collections.abc import AsyncIterator, Iterator
+from datetime import date
+from types import TracebackType
+from typing import Any, TypeVar
+
+import httpx
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+from . import _core as core
+from .errors import (
+    APIError,
+    DataFuelError,
+    NoAPIKey,
+    TransportError,
+    WaitTimeout,
+    api_error,
+)
+from .models import (
+    AIProvider,
+    Analytics,
+    BalanceSplit,
+    CancelResult,
+    Capabilities,
+    CrawlPage,
+    CrawlResult,
+    CrawlResultsPage,
+    CrawlStatus,
+    Health,
+    JobResults,
+    JobsPage,
+    JobStatus,
+    JSInstruction,
+    Profile,
+    ProtectionCheck,
+    Proxy,
+    ProxyCountry,
+    ProxyLocation,
+    Result,
+    SiteMap,
+    Status,
+    TasksPage,
+    TransactionsPage,
+)
+
+_Model = TypeVar("_Model", bound=BaseModel)
+
+
+class _Base:
+    """Configuration and wire formatting shared by both clients."""
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        *,
+        base_url: str = core.DEFAULT_BASE_URL,
+        timeout: float | None = core.DEFAULT_TIMEOUT,
+        max_retries: int = 2,
+        poll_interval: float = 2.0,
+        user_agent: str | None = None,
+    ) -> None:
+        self.api_key = api_key or os.environ.get("DATAFUEL_API_KEY", "")
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.max_retries = max(max_retries, 0)
+        # A non-positive interval would poll in a tight loop against the API.
+        self.poll_interval = poll_interval if poll_interval > 0 else 2.0
+        sdk = f"datafuel-py/{core.VERSION}"
+        self.user_agent = f"{user_agent} {sdk}" if user_agent else sdk
+
+    def _request_args(self, request: core.Request, timeout: float | None) -> dict[str, Any]:
+        if not self.api_key:
+            raise NoAPIKey("no API key: pass one to the client or set DATAFUEL_API_KEY")
+        return {
+            "method": request.method,
+            "url": self.base_url + request.path,
+            "params": request.params,
+            "json": request.body,
+            "headers": core.headers(self.api_key, self.user_agent, request),
+            "timeout": self.timeout if timeout is None else timeout,
+        }
+
+    @staticmethod
+    def _parse(response: httpx.Response, degraded_ok: bool = False) -> Any:
+        try:
+            body: Any = response.json() if response.content else None
+        except ValueError:
+            body = response.text
+        if (
+            degraded_ok
+            and response.status_code == 503
+            and isinstance(body, dict)
+            and "status" in body
+        ):
+            return body
+        if response.status_code >= 400:
+            raise api_error(response.status_code, body, _retry_after(response))
+        return body
+
+    def _budget(self, timeout: float | None) -> float:
+        """How long a synchronous call may keep waiting for its result."""
+        if timeout is not None:
+            return timeout
+        return self.timeout if self.timeout is not None else math.inf
+
+    @staticmethod
+    def _model(model: type[_Model], body: Any) -> _Model:
+        """Parse an answer, turning a surprise into a DataFuelError."""
+        try:
+            return model.model_validate(body)
+        except ValidationError as exc:
+            raise DataFuelError(f"unexpected answer from the API: {exc}") from exc
+
+    @staticmethod
+    def _models(model: type[_Model], body: Any) -> list[_Model]:
+        try:
+            return TypeAdapter(list[model]).validate_python(body)  # type: ignore[valid-type]
+        except ValidationError as exc:
+            raise DataFuelError(f"unexpected answer from the API: {exc}") from exc
+
+    @staticmethod
+    def _field(body: Any, name: str) -> Any:
+        if not isinstance(body, dict) or body.get(name) is None:
+            raise DataFuelError(f"unexpected answer from the API: no {name!r} field")
+        return body[name]
+
+    def _str_field(self, body: Any, name: str) -> str:
+        return str(self._field(body, name))
+
+    def _int_field(self, body: Any, name: str) -> int:
+        value = self._field(body, name)
+        try:
+            return int(value)
+        except (TypeError, ValueError) as exc:
+            raise DataFuelError(
+                f"unexpected answer from the API: {name!r} is not a number"
+            ) from exc
+
+
+def _retry_after(response: httpx.Response) -> float:
+    try:
+        return float(response.headers.get("Retry-After", 0))
+    except ValueError:
+        return 0.0
+
+
+#: The API rejects a longer key with 400; catching it here saves the round trip.
+MAX_IDEMPOTENCY_KEY = 255
+
+
+def _key(explicit: str | None) -> str:
+    if explicit and len(explicit) > MAX_IDEMPOTENCY_KEY:
+        raise ValueError(f"idempotency_key is longer than {MAX_IDEMPOTENCY_KEY} characters")
+    return explicit or core.new_idempotency_key()
+
+
+class DataFuel(_Base):
+    """Client for the DataFuel scraping API.
+
+    >>> df = DataFuel(os.environ["DATAFUEL_API_KEY"])
+    >>> df.markdown("https://example.com")
+
+    Pick the call by the shape of the work: one URL is :meth:`scrape`, a site's
+    URL list is :meth:`map`, many pages from a start URL is :meth:`crawl`, a
+    list of known URLs is :meth:`run_job`, a question for an AI engine is
+    :meth:`ask`, a Google search is :meth:`search`.
+
+    Every write carries an ``Idempotency-Key``, generated per request, so a
+    retry attaches to the task already running instead of charging twice.
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        *,
+        base_url: str = core.DEFAULT_BASE_URL,
+        timeout: float | None = core.DEFAULT_TIMEOUT,
+        max_retries: int = 2,
+        poll_interval: float = 2.0,
+        user_agent: str | None = None,
+        http_client: httpx.Client | None = None,
+    ) -> None:
+        super().__init__(
+            api_key,
+            base_url=base_url,
+            timeout=timeout,
+            max_retries=max_retries,
+            poll_interval=poll_interval,
+            user_agent=user_agent,
+        )
+        # Redirects stay off: httpx does not strip a custom header on a
+        # cross-host redirect, so X-API-Key would follow the target.
+        self._http = http_client or httpx.Client(follow_redirects=False)
+        self._owns_http = http_client is None
+
+    def __enter__(self) -> DataFuel:
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    def close(self) -> None:
+        if self._owns_http:
+            self._http.close()
+
+    # --- transport -----------------------------------------------------
+
+    def _send(self, request: core.Request, timeout: float | None = None) -> Any:
+        args = self._request_args(request, timeout)
+        for attempt in range(self.max_retries + 1):
+            try:
+                return self._parse(self._http.request(**args), request.degraded_ok)
+            except httpx.HTTPError as exc:
+                error: Exception = TransportError(str(exc))
+            except APIError as exc:
+                error = exc
+            last = attempt >= self.max_retries
+            if last or not request.retryable or not core.should_retry(error):
+                raise error
+            time.sleep(core.backoff(attempt, getattr(error, "retry_after", 0.0)))
+        raise AssertionError("unreachable")  # pragma: no cover
+
+    def _run_task(self, request: core.Request, timeout: float | None) -> Result:
+        deadline = time.monotonic() + self._budget(timeout)
+        attempt = 0
+        while True:
+            body = self._send(request, timeout)
+            if not core.still_processing(body):
+                break
+            delay = core.poll_delay(attempt)
+            if time.monotonic() + delay >= deadline:
+                raise WaitTimeout(
+                    "the task is still processing; re-send under the same "
+                    "idempotency key to pick it up",
+                    id=request.idempotency_key,
+                )
+            time.sleep(delay)
+            attempt += 1
+        result = self._model(Result, body)
+        result.raise_for_status()
+        return result
+
+    # --- pages ---------------------------------------------------------
+
+    def scrape(
+        self,
+        url: str,
+        *,
+        proxy: Proxy | None = None,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> Result:
+        """Fetch one URL and block until the result is ready.
+
+        Raises :class:`~datafuel.Blocked` when the target refused the request
+        and :class:`~datafuel.TaskFailed` otherwise; both carry the ``result``.
+
+        Options:
+            format: ``html`` (API default), ``markdown``, ``json``, ``png``, ``jpeg``;
+                see :class:`~datafuel.Format` for the rest.
+            js_rendering: render in a real browser. Slower, 5x credits on a
+                Basic proxy. Leave it off unless the page comes back empty.
+            wait_for / wait_for_timeout_ms: CSS selector to wait for.
+            js_instructions: browser actions run after load, a list of single-action
+                dicts run in order, e.g. ``[{"click": "#more"}, {"wait_ms": 1000}]``;
+                an action may repeat. A dict keyed by action is still accepted, but its
+                order is not guaranteed. :meth:`js_instructions` lists the actions.
+                Needs ``js_rendering``.
+            block_resource: resource type the browser must not load, e.g. ``Image``,
+                or a list of them.
+            main_content_only / include_images: markdown only.
+                ``include_images=False`` drops images and saves tokens.
+            extract: ``{"price": ".price", "links": "a @href"}`` — name to CSS
+                selector, ``" @attr"`` reads an attribute. Returns JSON.
+            extract_regex: the same with RE2 patterns on the raw HTML.
+            template: built-in extractors, comma separated: images, links,
+                headings, phone_numbers, emails, meta_tags, tables, schema_org, all.
+            method / body / content_type / headers / header_order / cookies:
+                the shape of the HTTP request sent to the target.
+            user_agent / user_agent_type: chrome, firefox, safari, edge.
+            ai: an :class:`~datafuel.AI` to post-process the page with an LLM.
+                :meth:`ai_providers` lists the providers and models.
+        """
+        request = core.build_scrape(
+            url, proxy=proxy, options=options, idempotency_key=_key(idempotency_key)
+        )
+        return self._run_task(request, timeout)
+
+    def markdown(self, url: str, **kwargs: Any) -> str:
+        """The one-liner: the page as LLM-ready markdown, images dropped."""
+        return self.scrape(url, format="markdown", include_images=False, **kwargs).text
+
+    def get_task(self, task_id: str, *, timeout: float | None = None) -> Result:
+        """Return a task by id, e.g. one created by a job or a crawl.
+
+        A task that is still running comes back with ``pending`` true.
+        """
+        body = self._send(core.Request("GET", f"/task/{core.path_segment(task_id)}"), timeout)
+        if core.still_processing(body):
+            return Result(id=task_id, status=Status.PROCESSING)
+        result = self._model(Result, body)
+        result.raise_for_status()
+        return result
+
+    def wait_task(self, task_id: str, *, timeout: float | None = None) -> Result:
+        """Poll a task until it is done. ``timeout`` None waits indefinitely.
+
+        Raises :class:`~datafuel.TaskFailed` or :class:`~datafuel.Blocked`
+        when the task failed.
+        """
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            result = self.get_task(task_id)
+            if not result.pending:
+                return result
+            if deadline is not None and time.monotonic() + self.poll_interval >= deadline:
+                raise WaitTimeout(f"task {task_id} is still running", id=task_id)
+            time.sleep(self.poll_interval)
+
+    def ask(
+        self,
+        prompt: str,
+        *,
+        engine: str,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> Result:
+        """Send a prompt to an AI engine and return its answer.
+
+        Args:
+            engine: openai, gemini, google_ai_mode, perplexity, copilot or duck_ai.
+                ``duck_ai`` answers with ``{text, citations}`` in ``Result.data``.
+            model: duck_ai only: gpt-6-luna (default), gpt-5.4-mini,
+                claude-haiku-4-5, mistral-small-2603, tinfoil/gpt-oss-120b or
+                tinfoil/gemma4-31b. Any other engine or model is a 400.
+            websearch: let the engine search the web. On duck_ai it only affects
+                the tinfoil/* models; other duck_ai models ignore it.
+            follow_up: a second prompt sent after the first answer. Only openai
+                and perplexity support it; other engines fail the task, refunded.
+            country: proxy country for the engine session.
+            location: where the engine should answer for; defaults to the exit country.
+            format: result format.
+        """
+        request = core.build_ask(
+            prompt, options={"engine": engine, **options}, idempotency_key=_key(idempotency_key)
+        )
+        return self._run_task(request, timeout)
+
+    def search(
+        self,
+        query: str,
+        *,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> Result:
+        """Run a Google search and return the results page.
+
+        10 credits per result page, whatever the proxy; pages Google did not
+        serve are refunded. ``Result.data`` holds the structured results with
+        the default ``json`` format, and its ``page`` is the number of pages
+        returned. With ``ai_overview=True`` it carries ``ai_overview``:
+        ``{text, products: [{title, price, source, rating, reviews, link}],
+        references: [{title, source, snippet, url, redirect}]}``.
+
+        Options:
+            country / language: Google's ``gl`` / ``hl``, e.g. ``us`` / ``en``.
+            page: number of result pages, 1 to 10 (more is capped at 10). They
+                come back merged into one ``organic`` list with running
+                positions; the other sections come from page 1.
+            google_domain: e.g. ``google.co.uk``. Default ``google.com``.
+            proxy_country: accepted but not used yet. Searches leave through
+                DataFuel's own pool; target a market with ``country`` and ``language``.
+            location / uule / lat + lon (+ radius): search origin, one of them.
+            cr, lr, tbs, safe (``active`` / ``off``), nfpr, filter, uds,
+                kgmid, si, ludocid, lsig, ibp: Google's own parameters.
+            device: ``desktop`` (default) or ``mobile``. Mobile results have
+                no ``total_results``.
+            ai_overview: also return Google's AI overview. Default ``False``.
+            format: ``json`` (default), ``markdown`` or ``html``.
+        """
+        request = core.build_search(query, options=options, idempotency_key=_key(idempotency_key))
+        return self._run_task(request, timeout)
+
+    def map(
+        self,
+        url: str,
+        *,
+        proxy: Proxy | None = None,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> SiteMap:
+        """List the URLs of a site without scraping them.
+
+        One credit per call on a Basic proxy, however many links come back.
+        Use it before a crawl to see how big a section is.
+
+        Args:
+            search: keep only links whose URL or title contains this.
+            sitemap: map only this sitemap, from a previous ``SiteMap.sitemaps``.
+            limit: default 5000, capped at 10000.
+            include_subdomains / ignore_sitemap / sitemap_only: discovery switches.
+        """
+        request = core.build_map(url, proxy=proxy, idempotency_key=_key(idempotency_key), **options)
+        task = self._run_task(request, timeout)
+        site = self._model(SiteMap, task.require_data())
+        site.task = task
+        return site
+
+    # --- crawl ---------------------------------------------------------
+
+    def start_crawl(
+        self,
+        url: str,
+        *,
+        proxy: Proxy | None = None,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a crawl and return its id immediately.
+
+        Follows links from the start URL and scrapes every page. Pages are
+        charged like single scrapes when they are queued; failed and blocked
+        pages are refunded.
+
+        Args:
+            max_pages: default 100, max 10000. The hard budget of the crawl.
+            max_depth: default 3, max 10. The start URL is depth 0.
+            include_paths / exclude_paths: RE2 on path?query; exclude wins.
+            include_subdomains / allow_backward_links: widen the default, which
+                follows only same-host URLs under the start URL's path.
+            concurrency: pages in flight, default 5.
+            **options: the page options of :meth:`scrape`, minus ``ai``.
+        """
+        crawl_options = {k: options.pop(k) for k in list(options) if k in _CRAWL_KEYS}
+        request = core.build_crawl(
+            url,
+            proxy=proxy,
+            options=options,
+            idempotency_key=_key(idempotency_key),
+            **crawl_options,
+        )
+        body = self._send(request, timeout)
+        return self._str_field(body, "job_id")
+
+    def get_crawl(self, crawl_id: str, *, timeout: float | None = None) -> CrawlStatus:
+        """Return the progress of a crawl."""
+        body = self._send(core.Request("GET", f"/crawl/{core.path_segment(crawl_id)}"), timeout)
+        return self._model(CrawlStatus, body)
+
+    def crawl_results(
+        self,
+        crawl_id: str,
+        *,
+        cursor: str | None = None,
+        limit: int | None = None,
+        timeout: float | None = None,
+    ) -> CrawlResultsPage:
+        """One page of results, in discovery order. ``limit`` 0 uses the API default."""
+        body = self._send(core.crawl_results_request(crawl_id, cursor, limit), timeout)
+        return self._model(CrawlResultsPage, body)
+
+    def crawl_pages(self, crawl_id: str, *, limit: int | None = None) -> Iterator[CrawlPage]:
+        """Iterate over every page of a crawl, fetching result pages as needed.
+
+        Readable while the crawl runs; unfinished pages report ``pending``.
+        """
+        cursor: str | None = None
+        seen: set[str] = set()
+        while True:
+            batch = self.crawl_results(crawl_id, cursor=cursor, limit=limit)
+            yield from batch.pages
+            cursor = batch.next_cursor
+            if not cursor:
+                return
+            if cursor in seen:
+                raise DataFuelError("the API repeated a crawl cursor; stopping to avoid a loop")
+            seen.add(cursor)
+
+    def wait_crawl(self, crawl_id: str, *, timeout: float | None = None) -> CrawlStatus:
+        """Poll until the crawl is done. ``timeout`` None waits indefinitely."""
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            status = self.get_crawl(crawl_id)
+            if status.done:
+                return status
+            if deadline is not None and time.monotonic() + self.poll_interval >= deadline:
+                raise WaitTimeout(f"crawl {crawl_id} is still running", id=crawl_id, status=status)
+            time.sleep(self.poll_interval)
+
+    def cancel_crawl(self, crawl_id: str, *, timeout: float | None = None) -> CancelResult:
+        """Stop a crawl. Pages not yet started are refunded.
+
+        Raises :class:`~datafuel.JobNotCancellable` when it already finished.
+        """
+        body = self._send(core.cancel_request("crawl", crawl_id), timeout)
+        return self._model(CancelResult, body)
+
+    def crawl(self, url: str, *, timeout: float | None = None, **kwargs: Any) -> CrawlResult:
+        """Start a crawl, wait for it, and return every page.
+
+        Check ``stop_reason``: ``insufficient_credits`` means it ended early.
+        For large crawls prefer :meth:`start_crawl` + :meth:`wait_crawl` +
+        :meth:`crawl_pages`, which stream instead of holding every page in
+        memory. On :class:`~datafuel.WaitTimeout` the exception carries the id:
+        the crawl keeps running and billing, so pick it up again.
+        """
+        crawl_id = self.start_crawl(url, **kwargs)
+        status = self.wait_crawl(crawl_id, timeout=timeout)
+        return CrawlResult(id=crawl_id, status=status, pages=list(self.crawl_pages(crawl_id)))
+
+    # --- jobs ----------------------------------------------------------
+
+    def create_job(
+        self,
+        urls: list[str],
+        *,
+        proxy: Proxy | None = None,
+        sequential: bool = False,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a batch of known URLs and return the job id.
+
+        Cheaper and more predictable than a crawl when you already have the
+        URLs. ``sequential`` runs them one after the other; the default runs
+        them concurrently, bounded by the account's concurrency limit.
+        """
+        request = core.build_job(
+            urls,
+            proxy=proxy,
+            sequential=sequential,
+            options=options,
+            idempotency_key=_key(idempotency_key),
+        )
+        body = self._send(request, timeout)
+        return self._str_field(body, "id")
+
+    def create_ask_job(
+        self,
+        prompts: list[str],
+        *,
+        engine: str,
+        sequential: bool = False,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a batch of prompts for an AI engine and return the job id."""
+        request = core.build_ask_job(
+            prompts,
+            sequential=sequential,
+            options={"engine": engine, **options},
+            idempotency_key=_key(idempotency_key),
+        )
+        body = self._send(request, timeout)
+        return self._str_field(body, "id")
+
+    def create_search_job(
+        self,
+        queries: list[str],
+        *,
+        sequential: bool = False,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a batch of Google searches and return the job id.
+
+        Takes the options of :meth:`search`. Needs at least two queries.
+        """
+        request = core.build_search_job(
+            queries,
+            sequential=sequential,
+            options=options,
+            idempotency_key=_key(idempotency_key),
+        )
+        body = self._send(request, timeout)
+        return self._str_field(body, "id")
+
+    def get_job(self, job_id: str, *, timeout: float | None = None) -> JobStatus:
+        """Return the progress of a job."""
+        body = self._send(core.Request("GET", f"/job/{core.path_segment(job_id)}"), timeout)
+        return self._model(JobStatus, body)
+
+    def job_results(self, job_id: str, *, timeout: float | None = None) -> JobResults:
+        """Return the per-task results of a job.
+
+        A job completes even when some of its tasks failed: check ``task.ok``
+        or call ``task.raise_for_status()`` per task.
+        """
+        body = self._send(core.Request("GET", f"/job/{core.path_segment(job_id)}/results"), timeout)
+        results = self._model(JobResults, body)
+        results.id = job_id
+        return results
+
+    def wait_job(self, job_id: str, *, timeout: float | None = None) -> JobStatus:
+        """Poll until the job is done. ``timeout`` None waits indefinitely."""
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            status = self.get_job(job_id)
+            if status.done:
+                return status
+            if deadline is not None and time.monotonic() + self.poll_interval >= deadline:
+                raise WaitTimeout(f"job {job_id} is still running", id=job_id, status=status)
+            time.sleep(self.poll_interval)
+
+    def run_job(
+        self, urls: list[str], *, timeout: float | None = None, **kwargs: Any
+    ) -> JobResults:
+        """Create a job, wait for it, and return its results."""
+        job_id = self.create_job(urls, **kwargs)
+        self.wait_job(job_id, timeout=timeout)
+        return self.job_results(job_id)
+
+    def run_ask_job(
+        self, prompts: list[str], *, engine: str, timeout: float | None = None, **kwargs: Any
+    ) -> JobResults:
+        """Create a prompt job, wait for it, and return its results."""
+        job_id = self.create_ask_job(prompts, engine=engine, **kwargs)
+        self.wait_job(job_id, timeout=timeout)
+        return self.job_results(job_id)
+
+    def run_search_job(
+        self, queries: list[str], *, timeout: float | None = None, **kwargs: Any
+    ) -> JobResults:
+        """Create a search job, wait for it, and return its results."""
+        job_id = self.create_search_job(queries, **kwargs)
+        self.wait_job(job_id, timeout=timeout)
+        return self.job_results(job_id)
+
+    def cancel_job(self, job_id: str, *, timeout: float | None = None) -> CancelResult:
+        """Stop a job. Tasks not yet started are refunded.
+
+        Raises :class:`~datafuel.JobNotCancellable` when it already finished.
+        """
+        body = self._send(core.cancel_request("job", job_id), timeout)
+        return self._model(CancelResult, body)
+
+    def list_jobs(
+        self,
+        *,
+        status: Status | str | None = None,
+        type: str | None = None,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        limit: int | None = None,
+        cursor: str | None = None,
+        timeout: float | None = None,
+    ) -> JobsPage:
+        """One page of your jobs and crawls, newest first.
+
+        Dates are UTC days, ``end_date`` inclusive. ``limit`` defaults to 50
+        (max 200). Pass ``next_cursor`` back as ``cursor`` until it is None.
+        """
+        request = core.list_request(
+            "/job",
+            status=status,
+            type=type,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+            cursor=cursor,
+        )
+        return self._model(JobsPage, self._send(request, timeout))
+
+    def list_tasks(
+        self,
+        *,
+        status: Status | str | None = None,
+        type: str | None = None,
+        job_id: str | None = None,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        limit: int | None = None,
+        cursor: str | None = None,
+        timeout: float | None = None,
+    ) -> TasksPage:
+        """One page of your tasks, newest first, including those of jobs and crawls.
+
+        Items carry no result: fetch it with :meth:`get_task`. Filters and
+        paging as in :meth:`list_jobs`, plus ``job_id``.
+        """
+        request = core.list_request(
+            "/task",
+            status=status,
+            type=type,
+            job_id=job_id,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+            cursor=cursor,
+        )
+        return self._model(TasksPage, self._send(request, timeout))
+
+    # --- account -------------------------------------------------------
+
+    def capabilities(self, *, timeout: float | None = None) -> Capabilities:
+        """Which task types and LLM engines are switched on right now."""
+        body = self._send(core.Request("GET", "/config/capabilities"), timeout)
+        return self._model(Capabilities, body)
+
+    def balance(self, *, timeout: float | None = None) -> int:
+        """Remaining credits: plan and pay-as-you-go together."""
+        body = self._send(core.Request("GET", "/users/@me/balance"), timeout)
+        return self._int_field(body, "balance")
+
+    def balance_split(self, *, timeout: float | None = None) -> BalanceSplit:
+        """Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits."""
+        body = self._send(core.Request("GET", "/users/@me/balance"), timeout)
+        return self._model(BalanceSplit, body)
+
+    def transactions(
+        self,
+        *,
+        operation: str | None = None,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        page: int | None = None,
+        limit: int | None = None,
+        timeout: float | None = None,
+    ) -> TransactionsPage:
+        """Credit movements, newest first: credit pack purchases, usage, refunds, expiry.
+
+        ``page`` starts at 1, ``limit`` defaults to 10 (max 200). ``sums``
+        totals each operation over the whole range, not just this page.
+        """
+        request = core.transactions_request(operation, start_date, end_date, page, limit)
+        return self._model(TransactionsPage, self._send(request, timeout))
+
+    def me(self, *, timeout: float | None = None) -> Profile:
+        """The account behind the API key."""
+        body = self._send(core.Request("GET", "/users/@me"), timeout)
+        return self._model(Profile, body)
+
+    def reset_api_key(self, *, timeout: float | None = None) -> str:
+        """Revoke the current key and return the new one.
+
+        This is the only time the new key is shown. The client keeps using the
+        revoked one: store the new key and build a new client with it.
+        """
+        body = self._send(core.Request("POST", "/users/@me/api-key/reset"), timeout)
+        return self._str_field(body, "api_key")
+
+    def analytics(
+        self,
+        *,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        interval: str | None = None,
+        module: str | None = None,
+        timeout: float | None = None,
+    ) -> Analytics:
+        """Task counts and credits over a date range, both ends inclusive.
+
+        Defaults to the last 30 days, ``daily``. ``interval``: hourly, daily,
+        weekly, monthly. ``module`` narrows it to one task type.
+        """
+        request = core.analytics_request(start_date, end_date, interval, module)
+        return self._model(Analytics, self._send(request, timeout))
+
+    def check_protection(
+        self, urls: list[str], *, timeout: float | None = None
+    ) -> list[ProtectionCheck]:
+        """Which anti-bot vendor sits in front of each URL. Nothing is scraped."""
+        body = self._send(core.Request("POST", "/filter/check", body=list(urls)), timeout)
+        return self._models(ProtectionCheck, body)
+
+    def proxy_locations(
+        self, proxy_type: str | None = None, *, timeout: float | None = None
+    ) -> list[ProxyCountry]:
+        """Countries, regions and cities a proxy plan can exit from."""
+        body = self._send(core.proxy_request("/config/proxy/locations", proxy_type), timeout)
+        return self._models(ProxyCountry, body)
+
+    def proxy_asns(
+        self, country: str, *, proxy_type: str | None = None, timeout: float | None = None
+    ) -> list[ProxyLocation]:
+        """ASNs a proxy plan can exit from in ``country`` (ISO 3166-1 alpha-2)."""
+        body = self._send(
+            core.proxy_request("/config/proxy/asn", proxy_type, country=country), timeout
+        )
+        return self._models(ProxyLocation, body)
+
+    def js_instructions(self, *, timeout: float | None = None) -> list[JSInstruction]:
+        """The browser actions ``js_instructions`` accepts, with an example each."""
+        body = self._send(core.Request("GET", "/config/js-instructions"), timeout)
+        return self._models(JSInstruction, self._field(body, "instructions"))
+
+    def ai_providers(self, *, timeout: float | None = None) -> list[AIProvider]:
+        """The LLM providers and models :class:`~datafuel.AI` accepts."""
+        body = self._send(core.Request("GET", "/config/ai-providers"), timeout)
+        return self._models(AIProvider, self._field(body, "providers"))
+
+    def health(self, *, deep: bool = False, timeout: float | None = None) -> Health:
+        """Whether the API is up. ``deep`` also checks the dependencies it needs to serve scrapes.
+
+        A degraded report is returned, not raised: read ``Health.ok``.
+        """
+        return self._model(Health, self._send(core.health_request(deep), timeout))
+
+
+_CRAWL_KEYS = frozenset(
+    {
+        "max_pages",
+        "max_depth",
+        "include_paths",
+        "exclude_paths",
+        "include_subdomains",
+        "allow_backward_links",
+        "concurrency",
+    }
+)
+
+
+class AsyncDataFuel(_Base):
+    """The asyncio client. Same methods as :class:`DataFuel`, awaited."""
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        *,
+        base_url: str = core.DEFAULT_BASE_URL,
+        timeout: float | None = core.DEFAULT_TIMEOUT,
+        max_retries: int = 2,
+        poll_interval: float = 2.0,
+        user_agent: str | None = None,
+        http_client: httpx.AsyncClient | None = None,
+    ) -> None:
+        super().__init__(
+            api_key,
+            base_url=base_url,
+            timeout=timeout,
+            max_retries=max_retries,
+            poll_interval=poll_interval,
+            user_agent=user_agent,
+        )
+        # Redirects stay off: httpx does not strip a custom header on a
+        # cross-host redirect, so X-API-Key would follow the target.
+        self._http = http_client or httpx.AsyncClient(follow_redirects=False)
+        self._owns_http = http_client is None
+
+    async def __aenter__(self) -> AsyncDataFuel:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        if self._owns_http:
+            await self._http.aclose()
+
+    # --- transport -----------------------------------------------------
+
+    async def _send(self, request: core.Request, timeout: float | None = None) -> Any:
+        args = self._request_args(request, timeout)
+        for attempt in range(self.max_retries + 1):
+            try:
+                return self._parse(await self._http.request(**args), request.degraded_ok)
+            except httpx.HTTPError as exc:
+                error: Exception = TransportError(str(exc))
+            except APIError as exc:
+                error = exc
+            last = attempt >= self.max_retries
+            if last or not request.retryable or not core.should_retry(error):
+                raise error
+            await asyncio.sleep(core.backoff(attempt, getattr(error, "retry_after", 0.0)))
+        raise AssertionError("unreachable")  # pragma: no cover
+
+    async def _run_task(self, request: core.Request, timeout: float | None) -> Result:
+        deadline = time.monotonic() + self._budget(timeout)
+        attempt = 0
+        while True:
+            body = await self._send(request, timeout)
+            if not core.still_processing(body):
+                break
+            delay = core.poll_delay(attempt)
+            if time.monotonic() + delay >= deadline:
+                raise WaitTimeout(
+                    "the task is still processing; re-send under the same "
+                    "idempotency key to pick it up",
+                    id=request.idempotency_key,
+                )
+            await asyncio.sleep(delay)
+            attempt += 1
+        result = self._model(Result, body)
+        result.raise_for_status()
+        return result
+
+    # --- pages ---------------------------------------------------------
+
+    async def scrape(
+        self,
+        url: str,
+        *,
+        proxy: Proxy | None = None,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> Result:
+        """Fetch one URL and wait for the result. See :meth:`DataFuel.scrape`."""
+        request = core.build_scrape(
+            url, proxy=proxy, options=options, idempotency_key=_key(idempotency_key)
+        )
+        return await self._run_task(request, timeout)
+
+    async def markdown(self, url: str, **kwargs: Any) -> str:
+        """The page as LLM-ready markdown, images dropped."""
+        result = await self.scrape(url, format="markdown", include_images=False, **kwargs)
+        return result.text
+
+    async def get_task(self, task_id: str, *, timeout: float | None = None) -> Result:
+        """Return a task by id. See :meth:`DataFuel.get_task`."""
+        body = await self._send(core.Request("GET", f"/task/{core.path_segment(task_id)}"), timeout)
+        if core.still_processing(body):
+            return Result(id=task_id, status=Status.PROCESSING)
+        result = self._model(Result, body)
+        result.raise_for_status()
+        return result
+
+    async def wait_task(self, task_id: str, *, timeout: float | None = None) -> Result:
+        """Poll a task until it is done. See :meth:`DataFuel.wait_task`."""
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            result = await self.get_task(task_id)
+            if not result.pending:
+                return result
+            if deadline is not None and time.monotonic() + self.poll_interval >= deadline:
+                raise WaitTimeout(f"task {task_id} is still running", id=task_id)
+            await asyncio.sleep(self.poll_interval)
+
+    async def ask(
+        self,
+        prompt: str,
+        *,
+        engine: str,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> Result:
+        """Send a prompt to an AI engine. See :meth:`DataFuel.ask`."""
+        request = core.build_ask(
+            prompt, options={"engine": engine, **options}, idempotency_key=_key(idempotency_key)
+        )
+        return await self._run_task(request, timeout)
+
+    async def search(
+        self,
+        query: str,
+        *,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> Result:
+        """Run a Google search. See :meth:`DataFuel.search`."""
+        request = core.build_search(query, options=options, idempotency_key=_key(idempotency_key))
+        return await self._run_task(request, timeout)
+
+    async def map(
+        self,
+        url: str,
+        *,
+        proxy: Proxy | None = None,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> SiteMap:
+        """List the URLs of a site. See :meth:`DataFuel.map`."""
+        request = core.build_map(url, proxy=proxy, idempotency_key=_key(idempotency_key), **options)
+        task = await self._run_task(request, timeout)
+        site = self._model(SiteMap, task.require_data())
+        site.task = task
+        return site
+
+    # --- crawl ---------------------------------------------------------
+
+    async def start_crawl(
+        self,
+        url: str,
+        *,
+        proxy: Proxy | None = None,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a crawl and return its id. See :meth:`DataFuel.start_crawl`."""
+        crawl_options = {k: options.pop(k) for k in list(options) if k in _CRAWL_KEYS}
+        request = core.build_crawl(
+            url,
+            proxy=proxy,
+            options=options,
+            idempotency_key=_key(idempotency_key),
+            **crawl_options,
+        )
+        body = await self._send(request, timeout)
+        return self._str_field(body, "job_id")
+
+    async def get_crawl(self, crawl_id: str, *, timeout: float | None = None) -> CrawlStatus:
+        """Return the progress of a crawl."""
+        body = await self._send(
+            core.Request("GET", f"/crawl/{core.path_segment(crawl_id)}"), timeout
+        )
+        return self._model(CrawlStatus, body)
+
+    async def crawl_results(
+        self,
+        crawl_id: str,
+        *,
+        cursor: str | None = None,
+        limit: int | None = None,
+        timeout: float | None = None,
+    ) -> CrawlResultsPage:
+        """One page of crawl results."""
+        body = await self._send(core.crawl_results_request(crawl_id, cursor, limit), timeout)
+        return self._model(CrawlResultsPage, body)
+
+    async def crawl_pages(
+        self, crawl_id: str, *, limit: int | None = None
+    ) -> AsyncIterator[CrawlPage]:
+        """Iterate over every page of a crawl, fetching result pages as needed."""
+        cursor: str | None = None
+        seen: set[str] = set()
+        while True:
+            batch = await self.crawl_results(crawl_id, cursor=cursor, limit=limit)
+            for page in batch.pages:
+                yield page
+            cursor = batch.next_cursor
+            if not cursor:
+                return
+            if cursor in seen:
+                raise DataFuelError("the API repeated a crawl cursor; stopping to avoid a loop")
+            seen.add(cursor)
+
+    async def wait_crawl(self, crawl_id: str, *, timeout: float | None = None) -> CrawlStatus:
+        """Poll until the crawl is done. ``timeout`` None waits indefinitely."""
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            status = await self.get_crawl(crawl_id)
+            if status.done:
+                return status
+            if deadline is not None and time.monotonic() + self.poll_interval >= deadline:
+                raise WaitTimeout(f"crawl {crawl_id} is still running", id=crawl_id, status=status)
+            await asyncio.sleep(self.poll_interval)
+
+    async def cancel_crawl(self, crawl_id: str, *, timeout: float | None = None) -> CancelResult:
+        """Stop a crawl. See :meth:`DataFuel.cancel_crawl`."""
+        body = await self._send(core.cancel_request("crawl", crawl_id), timeout)
+        return self._model(CancelResult, body)
+
+    async def crawl(self, url: str, *, timeout: float | None = None, **kwargs: Any) -> CrawlResult:
+        """Start a crawl, wait for it, and return every page."""
+        crawl_id = await self.start_crawl(url, **kwargs)
+        status = await self.wait_crawl(crawl_id, timeout=timeout)
+        pages = [page async for page in self.crawl_pages(crawl_id)]
+        return CrawlResult(id=crawl_id, status=status, pages=pages)
+
+    # --- jobs ----------------------------------------------------------
+
+    async def create_job(
+        self,
+        urls: list[str],
+        *,
+        proxy: Proxy | None = None,
+        sequential: bool = False,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a batch of known URLs. See :meth:`DataFuel.create_job`."""
+        request = core.build_job(
+            urls,
+            proxy=proxy,
+            sequential=sequential,
+            options=options,
+            idempotency_key=_key(idempotency_key),
+        )
+        body = await self._send(request, timeout)
+        return self._str_field(body, "id")
+
+    async def create_ask_job(
+        self,
+        prompts: list[str],
+        *,
+        engine: str,
+        sequential: bool = False,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a batch of prompts for an AI engine."""
+        request = core.build_ask_job(
+            prompts,
+            sequential=sequential,
+            options={"engine": engine, **options},
+            idempotency_key=_key(idempotency_key),
+        )
+        body = await self._send(request, timeout)
+        return self._str_field(body, "id")
+
+    async def create_search_job(
+        self,
+        queries: list[str],
+        *,
+        sequential: bool = False,
+        idempotency_key: str | None = None,
+        timeout: float | None = None,
+        **options: Any,
+    ) -> str:
+        """Queue a batch of Google searches. See :meth:`DataFuel.create_search_job`."""
+        request = core.build_search_job(
+            queries,
+            sequential=sequential,
+            options=options,
+            idempotency_key=_key(idempotency_key),
+        )
+        body = await self._send(request, timeout)
+        return self._str_field(body, "id")
+
+    async def get_job(self, job_id: str, *, timeout: float | None = None) -> JobStatus:
+        """Return the progress of a job."""
+        body = await self._send(core.Request("GET", f"/job/{core.path_segment(job_id)}"), timeout)
+        return self._model(JobStatus, body)
+
+    async def job_results(self, job_id: str, *, timeout: float | None = None) -> JobResults:
+        """Return the per-task results of a job."""
+        body = await self._send(
+            core.Request("GET", f"/job/{core.path_segment(job_id)}/results"), timeout
+        )
+        results = self._model(JobResults, body)
+        results.id = job_id
+        return results
+
+    async def wait_job(self, job_id: str, *, timeout: float | None = None) -> JobStatus:
+        """Poll until the job is done. ``timeout`` None waits indefinitely."""
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            status = await self.get_job(job_id)
+            if status.done:
+                return status
+            if deadline is not None and time.monotonic() + self.poll_interval >= deadline:
+                raise WaitTimeout(f"job {job_id} is still running", id=job_id, status=status)
+            await asyncio.sleep(self.poll_interval)
+
+    async def run_job(
+        self, urls: list[str], *, timeout: float | None = None, **kwargs: Any
+    ) -> JobResults:
+        """Create a job, wait for it, and return its results."""
+        job_id = await self.create_job(urls, **kwargs)
+        await self.wait_job(job_id, timeout=timeout)
+        return await self.job_results(job_id)
+
+    async def run_ask_job(
+        self, prompts: list[str], *, engine: str, timeout: float | None = None, **kwargs: Any
+    ) -> JobResults:
+        """Create a prompt job, wait for it, and return its results."""
+        job_id = await self.create_ask_job(prompts, engine=engine, **kwargs)
+        await self.wait_job(job_id, timeout=timeout)
+        return await self.job_results(job_id)
+
+    async def run_search_job(
+        self, queries: list[str], *, timeout: float | None = None, **kwargs: Any
+    ) -> JobResults:
+        """Create a search job, wait for it, and return its results."""
+        job_id = await self.create_search_job(queries, **kwargs)
+        await self.wait_job(job_id, timeout=timeout)
+        return await self.job_results(job_id)
+
+    async def cancel_job(self, job_id: str, *, timeout: float | None = None) -> CancelResult:
+        """Stop a job. See :meth:`DataFuel.cancel_job`."""
+        body = await self._send(core.cancel_request("job", job_id), timeout)
+        return self._model(CancelResult, body)
+
+    async def list_jobs(
+        self,
+        *,
+        status: Status | str | None = None,
+        type: str | None = None,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        limit: int | None = None,
+        cursor: str | None = None,
+        timeout: float | None = None,
+    ) -> JobsPage:
+        """One page of your jobs and crawls. See :meth:`DataFuel.list_jobs`."""
+        request = core.list_request(
+            "/job",
+            status=status,
+            type=type,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+            cursor=cursor,
+        )
+        return self._model(JobsPage, await self._send(request, timeout))
+
+    async def list_tasks(
+        self,
+        *,
+        status: Status | str | None = None,
+        type: str | None = None,
+        job_id: str | None = None,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        limit: int | None = None,
+        cursor: str | None = None,
+        timeout: float | None = None,
+    ) -> TasksPage:
+        """One page of your tasks. See :meth:`DataFuel.list_tasks`."""
+        request = core.list_request(
+            "/task",
+            status=status,
+            type=type,
+            job_id=job_id,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+            cursor=cursor,
+        )
+        return self._model(TasksPage, await self._send(request, timeout))
+
+    # --- account -------------------------------------------------------
+
+    async def capabilities(self, *, timeout: float | None = None) -> Capabilities:
+        """Which task types and LLM engines are switched on right now."""
+        body = await self._send(core.Request("GET", "/config/capabilities"), timeout)
+        return self._model(Capabilities, body)
+
+    async def balance(self, *, timeout: float | None = None) -> int:
+        """Remaining credits: plan and pay-as-you-go together."""
+        body = await self._send(core.Request("GET", "/users/@me/balance"), timeout)
+        return self._int_field(body, "balance")
+
+    async def balance_split(self, *, timeout: float | None = None) -> BalanceSplit:
+        """Remaining credits by pool. See :meth:`DataFuel.balance_split`."""
+        body = await self._send(core.Request("GET", "/users/@me/balance"), timeout)
+        return self._model(BalanceSplit, body)
+
+    async def transactions(
+        self,
+        *,
+        operation: str | None = None,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        page: int | None = None,
+        limit: int | None = None,
+        timeout: float | None = None,
+    ) -> TransactionsPage:
+        """Credit movements. See :meth:`DataFuel.transactions`."""
+        request = core.transactions_request(operation, start_date, end_date, page, limit)
+        return self._model(TransactionsPage, await self._send(request, timeout))
+
+    async def me(self, *, timeout: float | None = None) -> Profile:
+        """The account behind the API key."""
+        body = await self._send(core.Request("GET", "/users/@me"), timeout)
+        return self._model(Profile, body)
+
+    async def reset_api_key(self, *, timeout: float | None = None) -> str:
+        """Revoke the current key and return the new one."""
+        body = await self._send(core.Request("POST", "/users/@me/api-key/reset"), timeout)
+        return self._str_field(body, "api_key")
+
+    async def analytics(
+        self,
+        *,
+        start_date: date | str | None = None,
+        end_date: date | str | None = None,
+        interval: str | None = None,
+        module: str | None = None,
+        timeout: float | None = None,
+    ) -> Analytics:
+        """Task counts and credits over a date range. See :meth:`DataFuel.analytics`."""
+        request = core.analytics_request(start_date, end_date, interval, module)
+        return self._model(Analytics, await self._send(request, timeout))
+
+    async def check_protection(
+        self, urls: list[str], *, timeout: float | None = None
+    ) -> list[ProtectionCheck]:
+        """Which anti-bot vendor sits in front of each URL. Nothing is scraped."""
+        body = await self._send(core.Request("POST", "/filter/check", body=list(urls)), timeout)
+        return self._models(ProtectionCheck, body)
+
+    async def proxy_locations(
+        self, proxy_type: str | None = None, *, timeout: float | None = None
+    ) -> list[ProxyCountry]:
+        """Countries, regions and cities a proxy plan can exit from."""
+        body = await self._send(core.proxy_request("/config/proxy/locations", proxy_type), timeout)
+        return self._models(ProxyCountry, body)
+
+    async def proxy_asns(
+        self, country: str, *, proxy_type: str | None = None, timeout: float | None = None
+    ) -> list[ProxyLocation]:
+        """ASNs a proxy plan can exit from in ``country`` (ISO 3166-1 alpha-2)."""
+        body = await self._send(
+            core.proxy_request("/config/proxy/asn", proxy_type, country=country), timeout
+        )
+        return self._models(ProxyLocation, body)
+
+    async def js_instructions(self, *, timeout: float | None = None) -> list[JSInstruction]:
+        """The browser actions ``js_instructions`` accepts, with an example each."""
+        body = await self._send(core.Request("GET", "/config/js-instructions"), timeout)
+        return self._models(JSInstruction, self._field(body, "instructions"))
+
+    async def ai_providers(self, *, timeout: float | None = None) -> list[AIProvider]:
+        """The LLM providers and models :class:`~datafuel.AI` accepts."""
+        body = await self._send(core.Request("GET", "/config/ai-providers"), timeout)
+        return self._models(AIProvider, self._field(body, "providers"))
+
+    async def health(self, *, deep: bool = False, timeout: float | None = None) -> Health:
+        """Whether the API is up. See :meth:`DataFuel.health`."""
+        return self._model(Health, await self._send(core.health_request(deep), timeout))