firecrawl 4.12.0__py3-none-any.whl

firecrawl/v2/methods/aio/batch.py

@@ -0,0 +1,188 @@
+from typing import Optional, List, Dict, Any
+from ...types import ScrapeOptions, WebhookConfig, Document, BatchScrapeResponse, BatchScrapeJob, PaginationConfig
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.validation import prepare_scrape_options
+from ...utils.error_handler import handle_response_error
+from ...utils.normalize import normalize_document_input
+from ...methods.batch import validate_batch_urls
+import time
+
+def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
+    if not urls:
+        raise ValueError("URLs list cannot be empty")
+
+    validated_urls = validate_batch_urls([u.strip() if isinstance(u, str) else u for u in urls])
+    payload: Dict[str, Any] = {"urls": validated_urls}
+    if options:
+        opts = prepare_scrape_options(options)
+        if opts:
+            payload.update(opts)
+    if (w := kwargs.get("webhook")) is not None:
+        payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)
+    if (v := kwargs.get("append_to_id")) is not None:
+        payload["appendToId"] = v
+    if (v := kwargs.get("ignore_invalid_urls")) is not None:
+        payload["ignoreInvalidURLs"] = v
+    if (v := kwargs.get("max_concurrency")) is not None:
+        payload["maxConcurrency"] = v
+    if (v := kwargs.get("zero_data_retention")) is not None:
+        payload["zeroDataRetention"] = v
+    if (v := kwargs.get("integration")) is not None:
+        trimmed_integration = str(v).strip()
+        if trimmed_integration:
+            payload["integration"] = trimmed_integration
+    return payload
+
+
+async def start_batch_scrape(client: AsyncHttpClient, urls: List[str], **kwargs) -> BatchScrapeResponse:
+    payload = _prepare(urls, **kwargs)
+    response = await client.post("/v2/batch/scrape", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "start batch scrape")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    return BatchScrapeResponse(id=body.get("id"), url=body.get("url"), invalid_urls=body.get("invalidURLs"))
+
+
+async def get_batch_scrape_status(
+    client: AsyncHttpClient, 
+    job_id: str,
+    pagination_config: Optional[PaginationConfig] = None
+) -> BatchScrapeJob:
+    """
+    Get the status of a batch scrape job.
+    
+    Args:
+        client: Async HTTP client instance
+        job_id: ID of the batch scrape job
+        pagination_config: Optional configuration for pagination behavior
+        
+    Returns:
+        BatchScrapeJob containing job status and data
+        
+    Raises:
+        Exception: If the status check fails
+    """
+    response = await client.get(f"/v2/batch/scrape/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "get batch scrape status")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    docs: List[Document] = []
+    for doc in body.get("data", []) or []:
+        if isinstance(doc, dict):
+            normalized = normalize_document_input(doc)
+            docs.append(Document(**normalized))
+    
+    # Handle pagination if requested
+    auto_paginate = pagination_config.auto_paginate if pagination_config else True
+    if auto_paginate and body.get("next"):
+        docs = await _fetch_all_batch_pages_async(
+            client, 
+            body.get("next"), 
+            docs, 
+            pagination_config
+        )
+    
+    return BatchScrapeJob(
+        status=body.get("status"),
+        completed=body.get("completed", 0),
+        total=body.get("total", 0),
+        credits_used=body.get("creditsUsed"),
+        expires_at=body.get("expiresAt"),
+        next=body.get("next") if not auto_paginate else None,
+        data=docs,
+    )
+
+
+async def _fetch_all_batch_pages_async(
+    client: AsyncHttpClient,
+    next_url: str,
+    initial_documents: List[Document],
+    pagination_config: Optional[PaginationConfig] = None
+) -> List[Document]:
+    """
+    Fetch all pages of batch scrape results asynchronously.
+    
+    Args:
+        client: Async HTTP client instance
+        next_url: URL for the next page
+        initial_documents: Documents from the first page
+        pagination_config: Optional configuration for pagination limits
+        
+    Returns:
+        List of all documents from all pages
+    """
+    documents = initial_documents.copy()
+    current_url = next_url
+    page_count = 0
+    
+    # Apply pagination limits
+    max_pages = pagination_config.max_pages if pagination_config else None
+    max_results = pagination_config.max_results if pagination_config else None
+    max_wait_time = pagination_config.max_wait_time if pagination_config else None
+    
+    start_time = time.monotonic()
+    
+    while current_url:
+        # Check pagination limits
+        if (max_pages is not None) and (page_count >= max_pages):
+            break
+            
+        if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
+            break
+        
+        # Fetch next page
+        response = await client.get(current_url)
+        
+        if response.status_code >= 400:
+            # Log error but continue with what we have
+            import logging
+            logger = logging.getLogger("firecrawl")
+            logger.warning(f"Failed to fetch next page: {response.status_code}")
+            break
+        
+        page_data = response.json()
+        
+        if not page_data.get("success"):
+            break
+        
+        # Add documents from this page
+        for doc in page_data.get("data", []) or []:
+            if isinstance(doc, dict):
+                # Check max_results limit
+                if (max_results is not None) and (len(documents) >= max_results):
+                    break
+                normalized = normalize_document_input(doc)
+                documents.append(Document(**normalized))
+        
+        # Check if we hit max_results limit
+        if (max_results is not None) and (len(documents) >= max_results):
+            break
+        
+        # Get next URL
+        current_url = page_data.get("next")
+        page_count += 1
+    
+    return documents
+
+
+async def cancel_batch_scrape(client: AsyncHttpClient, job_id: str) -> bool:
+    response = await client.delete(f"/v2/batch/scrape/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "cancel batch scrape")
+    body = response.json()
+    return body.get("status") == "cancelled"
+
+
+async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[str, Any]:
+    response = await client.get(f"/v2/batch/scrape/{job_id}/errors")
+    if response.status_code >= 400:
+        handle_response_error(response, "get batch scrape errors")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    return body
+

firecrawl/v2/methods/aio/crawl.py

@@ -0,0 +1,351 @@
+from typing import Optional, Dict, Any, List
+from ...types import (
+    CrawlRequest,
+    CrawlJob,
+    CrawlResponse,
+    Document,
+    CrawlParamsRequest,
+    CrawlParamsData,
+    WebhookConfig,
+    CrawlErrorsResponse,
+    ActiveCrawlsResponse,
+    ActiveCrawl,
+    PaginationConfig,
+)
+from ...utils.error_handler import handle_response_error
+from ...utils.validation import prepare_scrape_options
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.normalize import normalize_document_input
+import time
+
+
+def _prepare_crawl_request(request: CrawlRequest) -> dict:
+    if not request.url or not request.url.strip():
+        raise ValueError("URL cannot be empty")
+    data = {"url": request.url}
+    if request.prompt:
+        data["prompt"] = request.prompt
+    if request.scrape_options is not None:
+        opts = prepare_scrape_options(request.scrape_options)
+        if opts:
+            data["scrapeOptions"] = opts
+    # Webhook conversion
+    if request.webhook is not None:
+        if isinstance(request.webhook, str):
+            data["webhook"] = request.webhook
+        else:
+            data["webhook"] = request.webhook.model_dump(exclude_none=True)
+    request_data = request.model_dump(exclude_none=True, exclude_unset=True)
+    request_data.pop("url", None)
+    request_data.pop("prompt", None)
+    request_data.pop("scrape_options", None)
+    field_mappings = {
+        "include_paths": "includePaths",
+        "exclude_paths": "excludePaths",
+        "max_discovery_depth": "maxDiscoveryDepth",
+        "ignore_sitemap": "ignoreSitemap",
+        "ignore_query_parameters": "ignoreQueryParameters",
+        "crawl_entire_domain": "crawlEntireDomain",
+        "allow_external_links": "allowExternalLinks",
+        "allow_subdomains": "allowSubdomains",
+        "delay": "delay",
+        "max_concurrency": "maxConcurrency",
+        "zero_data_retention": "zeroDataRetention",
+    }
+    for snake, camel in field_mappings.items():
+        if snake in request_data:
+            data[camel] = request_data.pop(snake)
+    data.update(request_data)
+    if getattr(request, "integration", None) is not None:
+        data["integration"] = str(getattr(request, "integration")).strip()
+    return data
+
+
+async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:
+    """
+    Start a crawl job for a website.
+    
+    Args:
+        client: Async HTTP client instance
+        request: CrawlRequest containing URL and options
+        
+    Returns:
+        CrawlResponse with job information
+        
+    Raises:
+        ValueError: If request is invalid
+        Exception: If the crawl operation fails to start
+    """
+    payload = _prepare_crawl_request(request)
+    response = await client.post("/v2/crawl", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "start crawl")
+    body = response.json()
+    if body.get("success"):
+        return CrawlResponse(id=body.get("id"), url=body.get("url"))
+    raise Exception(body.get("error", "Unknown error occurred"))
+
+
+async def get_crawl_status(
+    client: AsyncHttpClient,
+    job_id: str,
+    pagination_config: Optional[PaginationConfig] = None,
+    *,
+    request_timeout: Optional[float] = None,
+) -> CrawlJob:
+    """
+    Get the status of a crawl job.
+    
+    Args:
+        client: Async HTTP client instance
+        job_id: ID of the crawl job
+        pagination_config: Optional configuration for pagination limits
+        request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination 
+            is enabled (default) and there are multiple pages of results, this timeout applies to 
+            each page request separately, not to the entire operation
+        
+    Returns:
+        CrawlJob with job information
+        
+    Raises:
+        Exception: If the status check fails
+    """
+    response = await client.get(f"/v2/crawl/{job_id}", timeout=request_timeout)
+    if response.status_code >= 400:
+        handle_response_error(response, "get crawl status")
+    body = response.json()
+    if body.get("success"):
+        documents = []
+        for doc_data in body.get("data", []):
+            if isinstance(doc_data, dict):
+                normalized = normalize_document_input(doc_data)
+                documents.append(Document(**normalized))
+        
+        # Handle pagination if requested
+        auto_paginate = pagination_config.auto_paginate if pagination_config else True
+        if auto_paginate and body.get("next"):
+            documents = await _fetch_all_pages_async(
+                client,
+                body.get("next"),
+                documents,
+                pagination_config,
+                request_timeout=request_timeout,
+            )
+        
+        return CrawlJob(
+            status=body.get("status"),
+            completed=body.get("completed", 0),
+            total=body.get("total", 0),
+            credits_used=body.get("creditsUsed", 0),
+            expires_at=body.get("expiresAt"),
+            next=body.get("next") if not auto_paginate else None,
+            data=documents,
+        )
+    raise Exception(body.get("error", "Unknown error occurred"))
+
+
+async def _fetch_all_pages_async(
+    client: AsyncHttpClient,
+    next_url: str,
+    initial_documents: List[Document],
+    pagination_config: Optional[PaginationConfig] = None,
+    *,
+    request_timeout: Optional[float] = None,
+) -> List[Document]:
+    """
+    Fetch all pages of crawl results asynchronously.
+    
+    Args:
+        client: Async HTTP client instance
+        next_url: URL for the next page
+        initial_documents: Documents from the first page
+        pagination_config: Optional configuration for pagination limits
+        request_timeout: Optional timeout (in seconds) for the underlying HTTP request
+        
+    Returns:
+        List of all documents from all pages
+    """
+    documents = initial_documents.copy()
+    current_url = next_url
+    page_count = 0
+    
+    # Apply pagination limits
+    max_pages = pagination_config.max_pages if pagination_config else None
+    max_results = pagination_config.max_results if pagination_config else None
+    max_wait_time = pagination_config.max_wait_time if pagination_config else None
+    
+    start_time = time.monotonic()
+    
+    while current_url:
+        # Check pagination limits (treat 0 as a valid limit)
+        if (max_pages is not None) and page_count >= max_pages:
+            break
+
+        if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
+            break
+        
+        # Fetch next page
+        response = await client.get(current_url, timeout=request_timeout)
+        
+        if response.status_code >= 400:
+            # Log error but continue with what we have
+            import logging
+            logger = logging.getLogger("firecrawl")
+            logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
+            break
+        
+        page_data = response.json()
+        
+        if not page_data.get("success"):
+            break
+        
+        # Add documents from this page
+        for doc_data in page_data.get("data", []):
+            if isinstance(doc_data, dict):
+                # Check max_results limit
+                if (max_results is not None) and (len(documents) >= max_results):
+                    break
+                normalized = normalize_document_input(doc_data)
+                documents.append(Document(**normalized))
+        
+        # Check if we hit max_results limit
+        if (max_results is not None) and (len(documents) >= max_results):
+            break
+        
+        # Get next URL
+        current_url = page_data.get("next")
+        page_count += 1
+    
+    return documents
+
+
+async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:
+    """
+    Cancel a crawl job.
+    
+    Args:
+        client: Async HTTP client instance
+        job_id: ID of the crawl job
+        
+    Returns:
+        True if cancellation was successful
+        
+    Raises:
+        Exception: If the cancellation operation fails
+    """
+    response = await client.delete(f"/v2/crawl/{job_id}")
+    if response.status_code >= 400:
+        handle_response_error(response, "cancel crawl")
+    body = response.json()
+    return body.get("status") == "cancelled"
+
+
+async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
+    """
+    Preview crawl parameters before starting a crawl job.
+    
+    Args:
+        client: Async HTTP client instance
+        request: CrawlParamsRequest containing URL and prompt
+        
+    Returns:
+        CrawlParamsData containing crawl configuration
+        
+    Raises:
+        ValueError: If request is invalid
+        Exception: If the parameter preview fails
+    """
+    if not request.url or not request.url.strip():
+        raise ValueError("URL cannot be empty")
+    if not request.prompt or not request.prompt.strip():
+        raise ValueError("Prompt cannot be empty")
+    payload = {"url": request.url, "prompt": request.prompt}
+    response = await client.post("/v2/crawl/params-preview", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "crawl params preview")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    params_data = body.get("data", {})
+    converted: Dict[str, Any] = {}
+    mapping = {
+        "includePaths": "include_paths",
+        "excludePaths": "exclude_paths",
+        "maxDiscoveryDepth": "max_discovery_depth",
+        "ignoreSitemap": "ignore_sitemap",
+        "ignoreQueryParameters": "ignore_query_parameters",
+        "crawlEntireDomain": "crawl_entire_domain",
+        "allowExternalLinks": "allow_external_links",
+        "allowSubdomains": "allow_subdomains",
+        "maxConcurrency": "max_concurrency",
+        "scrapeOptions": "scrape_options",
+        "zeroDataRetention": "zero_data_retention",
+    }
+    for camel, snake in mapping.items():
+        if camel in params_data:
+            converted[snake] = params_data[camel]
+    if "webhook" in params_data:
+        wk = params_data["webhook"]
+        converted["webhook"] = wk
+    if "warning" in body:
+        converted["warning"] = body["warning"]
+    return CrawlParamsData(**converted)
+
+
+async def get_crawl_errors(client: AsyncHttpClient, crawl_id: str) -> CrawlErrorsResponse:
+    """
+    Get errors from a crawl job.
+    
+    Args:
+        client: Async HTTP client instance
+        crawl_id: ID of the crawl job
+        
+    Returns:
+        CrawlErrorsResponse with errors and robots blocked
+        
+    Raises:
+        Exception: If the error check operation fails
+    """
+    response = await client.get(f"/v2/crawl/{crawl_id}/errors")
+    if response.status_code >= 400:
+        handle_response_error(response, "check crawl errors")
+    body = response.json()
+    payload = body.get("data", body)
+    normalized = {
+        "errors": payload.get("errors", []),
+        "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
+    }
+    return CrawlErrorsResponse(**normalized)
+
+
+async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:
+    """
+    Get active crawl jobs.
+    
+    Args:
+        client: Async HTTP client instance
+        
+    Returns:
+        ActiveCrawlsResponse with active crawl jobs
+        
+    Raises:
+        Exception: If the active crawl jobs operation fails
+    """
+    response = await client.get("/v2/crawl/active")
+    if response.status_code >= 400:
+        handle_response_error(response, "get active crawls")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    crawls_in = body.get("crawls", [])
+    normalized = []
+    for c in crawls_in:
+        if isinstance(c, dict):
+            normalized.append({
+                "id": c.get("id"),
+                "team_id": c.get("teamId", c.get("team_id")),
+                "url": c.get("url"),
+                "options": c.get("options"),
+            })
+    return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])
+

firecrawl/v2/methods/aio/extract.py

@@ -0,0 +1,133 @@
+from typing import Any, Dict, List, Optional
+import asyncio
+
+from ...types import ExtractResponse, ScrapeOptions
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.validation import prepare_scrape_options
+
+
+def _prepare_extract_request(
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    integration: Optional[str] = None,
+) -> Dict[str, Any]:
+    body: Dict[str, Any] = {}
+    if urls is not None:
+        body["urls"] = urls
+    if prompt is not None:
+        body["prompt"] = prompt
+    if schema is not None:
+        body["schema"] = schema
+    if system_prompt is not None:
+        body["systemPrompt"] = system_prompt
+    if allow_external_links is not None:
+        body["allowExternalLinks"] = allow_external_links
+    if enable_web_search is not None:
+        body["enableWebSearch"] = enable_web_search
+    if show_sources is not None:
+        body["showSources"] = show_sources
+    if ignore_invalid_urls is not None:
+        body["ignoreInvalidURLs"] = ignore_invalid_urls
+    if scrape_options is not None:
+        prepared = prepare_scrape_options(scrape_options)
+        if prepared:
+            body["scrapeOptions"] = prepared
+    if integration is not None and str(integration).strip():
+        body["integration"] = str(integration).strip()
+    return body
+
+
+async def start_extract(
+    client: AsyncHttpClient,
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    integration: Optional[str] = None,
+) -> ExtractResponse:
+    body = _prepare_extract_request(
+        urls,
+        prompt=prompt,
+        schema=schema,
+        system_prompt=system_prompt,
+        allow_external_links=allow_external_links,
+        enable_web_search=enable_web_search,
+        show_sources=show_sources,
+        scrape_options=scrape_options,
+        ignore_invalid_urls=ignore_invalid_urls,
+        integration=integration,
+    )
+    resp = await client.post("/v2/extract", body)
+    return ExtractResponse(**resp.json())
+
+
+async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:
+    resp = await client.get(f"/v2/extract/{job_id}")
+    return ExtractResponse(**resp.json())
+
+
+async def wait_extract(
+    client: AsyncHttpClient,
+    job_id: str,
+    *,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None,
+) -> ExtractResponse:
+    start_ts = asyncio.get_event_loop().time()
+    while True:
+        status = await get_extract_status(client, job_id)
+        if status.status in ("completed", "failed", "cancelled"):
+            return status
+        if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
+            return status
+        await asyncio.sleep(max(1, poll_interval))
+
+
+async def extract(
+    client: AsyncHttpClient,
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None,
+    integration: Optional[str] = None,
+) -> ExtractResponse:
+    started = await start_extract(
+        client,
+        urls,
+        prompt=prompt,
+        schema=schema,
+        system_prompt=system_prompt,
+        allow_external_links=allow_external_links,
+        enable_web_search=enable_web_search,
+        show_sources=show_sources,
+        scrape_options=scrape_options,
+        ignore_invalid_urls=ignore_invalid_urls,
+        integration=integration,
+    )
+    job_id = getattr(started, "id", None)
+    if not job_id:
+        return started
+    return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
+