firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/v2/client_async.py

@@ -0,0 +1,250 @@
+"""
+Async v2 client mirroring the regular client surface using true async HTTP transport.
+"""
+
+import os
+import asyncio
+from typing import Optional, List, Dict, Any, Union, Callable, Literal
+from .types import (
+    ScrapeOptions,
+    CrawlRequest,
+    WebhookConfig,
+    SearchRequest,
+    SearchData,
+    SourceOption,
+    CrawlResponse,
+    CrawlJob,
+    CrawlParamsRequest,
+    CrawlParamsData,
+    CrawlErrorsResponse,
+    ActiveCrawlsResponse,
+    MapOptions,
+    MapData,
+    FormatOption,
+    WaitAction,
+    ScreenshotAction,
+    ClickAction,
+    WriteAction,
+    PressAction,
+    ScrollAction,
+    ScrapeAction,
+    ExecuteJavascriptAction,
+    PDFAction,
+    Location,
+)
+from .utils.http_client import HttpClient
+from .utils.http_client_async import AsyncHttpClient
+
+from .methods.aio import scrape as async_scrape  # type: ignore[attr-defined]
+from .methods.aio import batch as async_batch  # type: ignore[attr-defined]
+from .methods.aio import crawl as async_crawl  # type: ignore[attr-defined]
+from .methods.aio import search as async_search  # type: ignore[attr-defined]
+from .methods.aio import map as async_map # type: ignore[attr-defined]
+from .methods.aio import usage as async_usage # type: ignore[attr-defined]
+from .methods.aio import extract as async_extract  # type: ignore[attr-defined]
+
+from .watcher_async import AsyncWatcher
+
+class AsyncFirecrawlClient:
+    def __init__(self, api_key: Optional[str] = None, api_url: str = "https://api.firecrawl.dev"):
+        if api_key is None:
+            api_key = os.getenv("FIRECRAWL_API_KEY")
+        if not api_key:
+            raise ValueError("API key is required. Set FIRECRAWL_API_KEY or pass api_key.")
+        self.http_client = HttpClient(api_key, api_url)
+        self.async_http_client = AsyncHttpClient(api_key, api_url)
+
+    # Scrape
+    async def scrape(
+        self,
+        url: str,
+        **kwargs,
+    ):
+        options = ScrapeOptions(**{k: v for k, v in kwargs.items() if v is not None}) if kwargs else None
+        return await async_scrape.scrape(self.async_http_client, url, options)
+
+    # Search
+    async def search(
+        self,
+        query: str,
+        **kwargs,
+    ) -> SearchData:
+        request = SearchRequest(query=query, **{k: v for k, v in kwargs.items() if v is not None})
+        return await async_search.search(self.async_http_client, request)
+
+    async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:
+        request = CrawlRequest(url=url, **kwargs)
+        return await async_crawl.start_crawl(self.async_http_client, request)
+
+    async def wait_crawl(self, job_id: str, poll_interval: int = 2, timeout: Optional[int] = None) -> CrawlJob:
+        # simple polling loop using blocking get (ok for test-level async)
+        start = asyncio.get_event_loop().time()
+        while True:
+            status = await async_crawl.get_crawl_status(self.async_http_client, job_id)
+            if status.status in ["completed", "failed"]:
+                return status
+            if timeout and (asyncio.get_event_loop().time() - start) > timeout:
+                raise TimeoutError("Crawl wait timed out")
+            await asyncio.sleep(poll_interval)
+
+    async def crawl(self, **kwargs) -> CrawlJob:
+        # wrapper combining start and wait
+        resp = await self.start_crawl(**{k: v for k, v in kwargs.items() if k not in ("poll_interval", "timeout")})
+        poll_interval = kwargs.get("poll_interval", 2)
+        timeout = kwargs.get("timeout")
+        return await self.wait_crawl(resp.id, poll_interval=poll_interval, timeout=timeout)
+
+    async def get_crawl_status(self, job_id: str) -> CrawlJob:
+        return await async_crawl.get_crawl_status(self.async_http_client, job_id)
+
+    async def cancel_crawl(self, job_id: str) -> bool:
+        return await async_crawl.cancel_crawl(self.async_http_client, job_id)
+
+    async def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:
+        req = CrawlParamsRequest(url=url, prompt=prompt)
+        return await async_crawl.crawl_params_preview(self.async_http_client, req)
+
+    async def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:
+        return await async_crawl.get_crawl_errors(self.async_http_client, crawl_id)
+
+    async def get_active_crawls(self) -> ActiveCrawlsResponse:
+        return await async_crawl.get_active_crawls(self.async_http_client)
+
+    async def active_crawls(self) -> ActiveCrawlsResponse:
+        return await self.get_active_crawls()
+
+    # Map
+    async def map(
+        self,
+        url: str,
+        *,
+        search: Optional[str] = None,
+        include_subdomains: Optional[bool] = None,
+        limit: Optional[int] = None,
+        sitemap: Optional[Literal["only", "include", "skip"]] = None,
+        timeout: Optional[int] = None,
+    ) -> MapData:
+        options = MapOptions(
+            search=search,
+            include_subdomains=include_subdomains,
+            limit=limit,
+            sitemap=sitemap if sitemap is not None else "include",
+            timeout=timeout,
+        ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, timeout]) else None
+        return await async_map.map(self.async_http_client, url, options)
+
+    async def start_batch_scrape(self, urls: List[str], **kwargs) -> Any:
+        return await async_batch.start_batch_scrape(self.async_http_client, urls, **kwargs)
+
+    async def wait_batch_scrape(self, job_id: str, poll_interval: int = 2, timeout: Optional[int] = None) -> Any:
+        start = asyncio.get_event_loop().time()
+        while True:
+            status = await async_batch.get_batch_scrape_status(self.async_http_client, job_id)
+            if status.status in ["completed", "failed", "cancelled"]:
+                return status
+            if timeout and (asyncio.get_event_loop().time() - start) > timeout:
+                raise TimeoutError("Batch wait timed out")
+            await asyncio.sleep(poll_interval)
+
+    async def batch_scrape(self, urls: List[str], **kwargs) -> Any:
+        # waiter wrapper
+        start = await self.start_batch_scrape(urls, **{k: v for k, v in kwargs.items() if k not in ("poll_interval", "timeout")})
+        job_id = start.id
+        poll_interval = kwargs.get("poll_interval", 2)
+        timeout = kwargs.get("timeout")
+        return await self.wait_batch_scrape(job_id, poll_interval=poll_interval, timeout=timeout)
+
+    async def get_batch_scrape_status(self, job_id: str):
+        return await async_batch.get_batch_scrape_status(self.async_http_client, job_id)
+
+    async def cancel_batch_scrape(self, job_id: str) -> bool:
+        return await async_batch.cancel_batch_scrape(self.async_http_client, job_id)
+
+    async def get_batch_scrape_errors(self, job_id: str) -> CrawlErrorsResponse:
+        # Returns v2 errors structure; typed as CrawlErrorsResponse for parity
+        return await async_batch.get_batch_scrape_errors(self.async_http_client, job_id)  # type: ignore[return-value]
+
+    # Extract (proxy to v1 async)
+    async def extract(
+        self,
+        urls: Optional[List[str]] = None,
+        *,
+        prompt: Optional[str] = None,
+        schema: Optional[Dict[str, Any]] = None,
+        system_prompt: Optional[str] = None,
+        allow_external_links: Optional[bool] = None,
+        enable_web_search: Optional[bool] = None,
+        show_sources: Optional[bool] = None,
+        scrape_options: Optional['ScrapeOptions'] = None,
+        ignore_invalid_urls: Optional[bool] = None,
+        poll_interval: int = 2,
+        timeout: Optional[int] = None,
+    ):
+        return await async_extract.extract(
+            self.async_http_client,
+            urls,
+            prompt=prompt,
+            schema=schema,
+            system_prompt=system_prompt,
+            allow_external_links=allow_external_links,
+            enable_web_search=enable_web_search,
+            show_sources=show_sources,
+            scrape_options=scrape_options,
+            ignore_invalid_urls=ignore_invalid_urls,
+            poll_interval=poll_interval,
+            timeout=timeout,
+        )
+
+    async def get_extract_status(self, job_id: str):
+        return await async_extract.get_extract_status(self.async_http_client, job_id)
+
+    async def start_extract(
+        self,
+        urls: Optional[List[str]] = None,
+        *,
+        prompt: Optional[str] = None,
+        schema: Optional[Dict[str, Any]] = None,
+        system_prompt: Optional[str] = None,
+        allow_external_links: Optional[bool] = None,
+        enable_web_search: Optional[bool] = None,
+        show_sources: Optional[bool] = None,
+        scrape_options: Optional['ScrapeOptions'] = None,
+        ignore_invalid_urls: Optional[bool] = None,
+    ):
+        return await async_extract.start_extract(
+            self.async_http_client,
+            urls,
+            prompt=prompt,
+            schema=schema,
+            system_prompt=system_prompt,
+            allow_external_links=allow_external_links,
+            enable_web_search=enable_web_search,
+            show_sources=show_sources,
+            scrape_options=scrape_options,
+            ignore_invalid_urls=ignore_invalid_urls,
+        )
+
+    # Usage endpoints
+    async def get_concurrency(self):
+        from .methods.aio import usage as async_usage  # type: ignore[attr-defined]
+        return await async_usage.get_concurrency(self.async_http_client)
+
+    async def get_credit_usage(self):
+        from .methods.aio import usage as async_usage  # type: ignore[attr-defined]
+        return await async_usage.get_credit_usage(self.async_http_client)
+
+    async def get_token_usage(self):
+        from .methods.aio import usage as async_usage  # type: ignore[attr-defined]
+        return await async_usage.get_token_usage(self.async_http_client)
+
+    # Watcher (sync object usable from async contexts)
+    def watcher(
+        self,
+        job_id: str,
+        *,
+        kind: Literal["crawl", "batch"] = "crawl",
+        poll_interval: int = 2,
+        timeout: Optional[int] = None,
+    ) -> AsyncWatcher:
+        return AsyncWatcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout)
+

firecrawl/v2/methods/aio/__init__.py

@@ -0,0 +1 @@
+# Async (aio) method modules for v2

firecrawl/v2/methods/aio/batch.py

@@ -0,0 +1,85 @@
+from typing import Optional, List, Dict, Any
+from ...types import ScrapeOptions, WebhookConfig, Document, BatchScrapeResponse, BatchScrapeJob
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.validation import prepare_scrape_options
+from ...utils.error_handler import handle_response_error
+
+
+def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
+    if not urls:
+        raise ValueError("URLs list cannot be empty")
+    payload: Dict[str, Any] = {"urls": [u.strip() for u in urls]}
+    if options:
+        opts = prepare_scrape_options(options)
+        if opts:
+            payload.update(opts)
+    if (w := kwargs.get("webhook")) is not None:
+        payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)
+    if (v := kwargs.get("append_to_id")) is not None:
+        payload["appendToId"] = v
+    if (v := kwargs.get("ignore_invalid_urls")) is not None:
+        payload["ignoreInvalidURLs"] = v
+    if (v := kwargs.get("max_concurrency")) is not None:
+        payload["maxConcurrency"] = v
+    if (v := kwargs.get("zero_data_retention")) is not None:
+        payload["zeroDataRetention"] = v
+    if (v := kwargs.get("integration")) is not None:
+        payload["integration"] = v
+    return payload
+
+
+async def start_batch_scrape(client: AsyncHttpClient, urls: List[str], **kwargs) -> BatchScrapeResponse:
+    payload = _prepare(urls, **kwargs)
+    response = await client.post("/v2/batch/scrape", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "start batch scrape")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    return BatchScrapeResponse(id=body.get("id"), url=body.get("url"), invalid_urls=body.get("invalidURLs"))
+
+
+async def get_batch_scrape_status(client: AsyncHttpClient, job_id: str) -> BatchScrapeJob:
+    response = await client.get(f"/v2/batch/scrape/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "get batch scrape status")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    docs: List[Document] = []
+    for doc in body.get("data", []) or []:
+        if isinstance(doc, dict):
+            normalized = dict(doc)
+            if 'rawHtml' in normalized and 'raw_html' not in normalized:
+                normalized['raw_html'] = normalized.pop('rawHtml')
+            if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+                normalized['change_tracking'] = normalized.pop('changeTracking')
+            docs.append(Document(**normalized))
+    return BatchScrapeJob(
+        status=body.get("status"),
+        completed=body.get("completed", 0),
+        total=body.get("total", 0),
+        credits_used=body.get("creditsUsed"),
+        expires_at=body.get("expiresAt"),
+        next=body.get("next"),
+        data=docs,
+    )
+
+
+async def cancel_batch_scrape(client: AsyncHttpClient, job_id: str) -> bool:
+    response = await client.delete(f"/v2/batch/scrape/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "cancel batch scrape")
+    body = response.json()
+    return body.get("status") == "cancelled"
+
+
+async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[str, Any]:
+    response = await client.get(f"/v2/batch/scrape/{job_id}/errors")
+    if response.status_code >= 400:
+        handle_response_error(response, "get batch scrape errors")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    return body
+

firecrawl/v2/methods/aio/crawl.py

@@ -0,0 +1,174 @@
+from typing import Optional, Dict, Any
+from ...types import (
+    CrawlRequest,
+    CrawlJob,
+    CrawlResponse,
+    Document,
+    CrawlParamsRequest,
+    CrawlParamsData,
+    WebhookConfig,
+    CrawlErrorsResponse,
+    ActiveCrawlsResponse,
+    ActiveCrawl,
+)
+from ...utils.error_handler import handle_response_error
+from ...utils.validation import prepare_scrape_options
+from ...utils.http_client_async import AsyncHttpClient
+
+
+def _prepare_crawl_request(request: CrawlRequest) -> dict:
+    if not request.url or not request.url.strip():
+        raise ValueError("URL cannot be empty")
+    data = {"url": request.url}
+    if request.prompt:
+        data["prompt"] = request.prompt
+    if request.scrape_options is not None:
+        opts = prepare_scrape_options(request.scrape_options)
+        if opts:
+            data["scrapeOptions"] = opts
+    # Webhook conversion
+    if request.webhook is not None:
+        if isinstance(request.webhook, str):
+            data["webhook"] = request.webhook
+        else:
+            data["webhook"] = request.webhook.model_dump(exclude_none=True)
+    request_data = request.model_dump(exclude_none=True, exclude_unset=True)
+    request_data.pop("url", None)
+    request_data.pop("prompt", None)
+    request_data.pop("scrape_options", None)
+    field_mappings = {
+        "include_paths": "includePaths",
+        "exclude_paths": "excludePaths",
+        "max_discovery_depth": "maxDiscoveryDepth",
+        "ignore_sitemap": "ignoreSitemap",
+        "ignore_query_parameters": "ignoreQueryParameters",
+        "crawl_entire_domain": "crawlEntireDomain",
+        "allow_external_links": "allowExternalLinks",
+        "allow_subdomains": "allowSubdomains",
+        "delay": "delay",
+        "max_concurrency": "maxConcurrency",
+        "zero_data_retention": "zeroDataRetention",
+    }
+    for snake, camel in field_mappings.items():
+        if snake in request_data:
+            data[camel] = request_data.pop(snake)
+    data.update(request_data)
+    return data
+
+
+async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:
+    payload = _prepare_crawl_request(request)
+    response = await client.post("/v2/crawl", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "start crawl")
+    body = response.json()
+    if body.get("success"):
+        return CrawlResponse(id=body.get("id"), url=body.get("url"))
+    raise Exception(body.get("error", "Unknown error occurred"))
+
+
+async def get_crawl_status(client: AsyncHttpClient, job_id: str) -> CrawlJob:
+    response = await client.get(f"/v2/crawl/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "get crawl status")
+    body = response.json()
+    if body.get("success"):
+        documents = []
+        for doc_data in body.get("data", []):
+            if isinstance(doc_data, dict):
+                normalized = dict(doc_data)
+                if 'rawHtml' in normalized and 'raw_html' not in normalized:
+                    normalized['raw_html'] = normalized.pop('rawHtml')
+                if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+                    normalized['change_tracking'] = normalized.pop('changeTracking')
+                documents.append(Document(**normalized))
+        return CrawlJob(
+            status=body.get("status"),
+            completed=body.get("completed", 0),
+            total=body.get("total", 0),
+            credits_used=body.get("creditsUsed", 0),
+            expires_at=body.get("expiresAt"),
+            next=body.get("next"),
+            data=documents,
+        )
+    raise Exception(body.get("error", "Unknown error occurred"))
+
+
+async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:
+    response = await client.delete(f"/v2/crawl/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "cancel crawl")
+    body = response.json()
+    return body.get("status") == "cancelled"
+
+
+async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
+    if not request.url or not request.url.strip():
+        raise ValueError("URL cannot be empty")
+    if not request.prompt or not request.prompt.strip():
+        raise ValueError("Prompt cannot be empty")
+    payload = {"url": request.url, "prompt": request.prompt}
+    response = await client.post("/v2/crawl/params-preview", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "crawl params preview")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    params_data = body.get("data", {})
+    converted: Dict[str, Any] = {}
+    mapping = {
+        "includePaths": "include_paths",
+        "excludePaths": "exclude_paths",
+        "maxDiscoveryDepth": "max_discovery_depth",
+        "ignoreSitemap": "ignore_sitemap",
+        "ignoreQueryParameters": "ignore_query_parameters",
+        "crawlEntireDomain": "crawl_entire_domain",
+        "allowExternalLinks": "allow_external_links",
+        "allowSubdomains": "allow_subdomains",
+        "maxConcurrency": "max_concurrency",
+        "scrapeOptions": "scrape_options",
+        "zeroDataRetention": "zero_data_retention",
+    }
+    for camel, snake in mapping.items():
+        if camel in params_data:
+            converted[snake] = params_data[camel]
+    if "webhook" in params_data:
+        wk = params_data["webhook"]
+        converted["webhook"] = wk
+    if "warning" in body:
+        converted["warning"] = body["warning"]
+    return CrawlParamsData(**converted)
+
+
+async def get_crawl_errors(client: AsyncHttpClient, crawl_id: str) -> CrawlErrorsResponse:
+    response = await client.get(f"/v2/crawl/{crawl_id}/errors")
+    if response.status_code >= 400:
+        handle_response_error(response, "check crawl errors")
+    body = response.json()
+    payload = body.get("data", body)
+    normalized = {
+        "errors": payload.get("errors", []),
+        "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
+    }
+    return CrawlErrorsResponse(**normalized)
+
+
+async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:
+    response = await client.get("/v2/crawl/active")
+    if response.status_code >= 400:
+        handle_response_error(response, "get active crawls")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    crawls_in = body.get("crawls", [])
+    normalized = []
+    for c in crawls_in:
+        if isinstance(c, dict):
+            normalized.append({
+                "id": c.get("id"),
+                "team_id": c.get("teamId", c.get("team_id")),
+                "url": c.get("url"),
+                "options": c.get("options"),
+            })
+    return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])
+

firecrawl/v2/methods/aio/extract.py

@@ -0,0 +1,126 @@
+from typing import Any, Dict, List, Optional
+import asyncio
+
+from ...types import ExtractResponse, ScrapeOptions
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.validation import prepare_scrape_options
+
+
+def _prepare_extract_request(
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+) -> Dict[str, Any]:
+    body: Dict[str, Any] = {}
+    if urls is not None:
+        body["urls"] = urls
+    if prompt is not None:
+        body["prompt"] = prompt
+    if schema is not None:
+        body["schema"] = schema
+    if system_prompt is not None:
+        body["systemPrompt"] = system_prompt
+    if allow_external_links is not None:
+        body["allowExternalLinks"] = allow_external_links
+    if enable_web_search is not None:
+        body["enableWebSearch"] = enable_web_search
+    if show_sources is not None:
+        body["showSources"] = show_sources
+    if ignore_invalid_urls is not None:
+        body["ignoreInvalidURLs"] = ignore_invalid_urls
+    if scrape_options is not None:
+        prepared = prepare_scrape_options(scrape_options)
+        if prepared:
+            body["scrapeOptions"] = prepared
+    return body
+
+
+async def start_extract(
+    client: AsyncHttpClient,
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+) -> ExtractResponse:
+    body = _prepare_extract_request(
+        urls,
+        prompt=prompt,
+        schema=schema,
+        system_prompt=system_prompt,
+        allow_external_links=allow_external_links,
+        enable_web_search=enable_web_search,
+        show_sources=show_sources,
+        scrape_options=scrape_options,
+        ignore_invalid_urls=ignore_invalid_urls,
+    )
+    resp = await client.post("/v2/extract", body)
+    return ExtractResponse(**resp.json())
+
+
+async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:
+    resp = await client.get(f"/v2/extract/{job_id}")
+    return ExtractResponse(**resp.json())
+
+
+async def wait_extract(
+    client: AsyncHttpClient,
+    job_id: str,
+    *,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None,
+) -> ExtractResponse:
+    start_ts = asyncio.get_event_loop().time()
+    while True:
+        status = await get_extract_status(client, job_id)
+        if status.status in ("completed", "failed", "cancelled"):
+            return status
+        if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
+            return status
+        await asyncio.sleep(max(1, poll_interval))
+
+
+async def extract(
+    client: AsyncHttpClient,
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None,
+) -> ExtractResponse:
+    started = await start_extract(
+        client,
+        urls,
+        prompt=prompt,
+        schema=schema,
+        system_prompt=system_prompt,
+        allow_external_links=allow_external_links,
+        enable_web_search=enable_web_search,
+        show_sources=show_sources,
+        scrape_options=scrape_options,
+        ignore_invalid_urls=ignore_invalid_urls,
+    )
+    job_id = getattr(started, "id", None)
+    if not job_id:
+        return started
+    return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
+