firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/v2/methods/aio/map.py

@@ -0,0 +1,59 @@
+from typing import Optional, Dict, Any
+from ...types import MapOptions, MapData, LinkResult
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.error_handler import handle_response_error
+
+
+def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
+    if not url or not url.strip():
+        raise ValueError("URL cannot be empty")
+    payload: Dict[str, Any] = {"url": url.strip()}
+    if options is not None:
+        data: Dict[str, Any] = {}
+        if getattr(options, "sitemap", None) is not None:
+            data["sitemap"] = options.sitemap
+        if options.search is not None:
+            data["search"] = options.search
+        if options.include_subdomains is not None:
+            data["includeSubdomains"] = options.include_subdomains
+        if options.limit is not None:
+            data["limit"] = options.limit
+        if options.timeout is not None:
+            data["timeout"] = options.timeout
+        payload.update(data)
+    return payload
+
+
+async def map(client: AsyncHttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
+    request_data = _prepare_map_request(url, options)
+    response = await client.post("/v2/map", request_data)
+    if response.status_code >= 400:
+        handle_response_error(response, "map")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    
+    
+    # data = body.get("data", {})
+    # result_links: list[LinkResult] = []
+    # for item in data.get("links", []):
+    #     if isinstance(item, dict):
+    #         result_links.append(
+    #             LinkResult(
+    #                 url=item.get("url", ""),
+    #                 title=item.get("title"),
+    #                 description=item.get("description"),
+    #             )
+    #         )
+    #     elif isinstance(item, str):
+    #         result_links.append(LinkResult(url=item))
+
+    result_links: list[LinkResult] = []
+    for item in body.get("links", []):
+        if isinstance(item, dict):
+            result_links.append(LinkResult(url=item.get("url", ""), title=item.get("title"), description=item.get("description")))
+        elif isinstance(item, str):
+            result_links.append(LinkResult(url=item))
+
+    return MapData(links=result_links)
+

firecrawl/v2/methods/aio/scrape.py

@@ -0,0 +1,36 @@
+from typing import Optional, Dict, Any
+from ...types import ScrapeOptions, Document
+from ...utils.error_handler import handle_response_error
+from ...utils.validation import prepare_scrape_options, validate_scrape_options
+from ...utils.http_client_async import AsyncHttpClient
+
+
+async def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
+    if not url or not url.strip():
+        raise ValueError("URL cannot be empty")
+    payload: Dict[str, Any] = {"url": url.strip()}
+    if options is not None:
+        validated = validate_scrape_options(options)
+        if validated is not None:
+            opts = prepare_scrape_options(validated)
+            if opts:
+                payload.update(opts)
+    return payload
+
+
+async def scrape(client: AsyncHttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
+    payload = await _prepare_scrape_request(url, options)
+    response = await client.post("/v2/scrape", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "scrape")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    document_data = body.get("data", {})
+    normalized = dict(document_data)
+    if 'rawHtml' in normalized and 'raw_html' not in normalized:
+        normalized['raw_html'] = normalized.pop('rawHtml')
+    if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+        normalized['change_tracking'] = normalized.pop('changeTracking')
+    return Document(**normalized)
+

firecrawl/v2/methods/aio/search.py

@@ -0,0 +1,58 @@
+from typing import Dict, Any
+from ...types import SearchRequest, SearchData, SearchResult, Document
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.error_handler import handle_response_error
+from ...utils.validation import prepare_scrape_options, validate_scrape_options
+
+
+def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
+    data = request.model_dump(exclude_none=True)
+    if request.ignore_invalid_urls is not None:
+        data["ignoreInvalidURLs"] = request.ignore_invalid_urls
+        data.pop("ignore_invalid_urls", None)
+    if request.scrape_options is not None:
+        validate_scrape_options(request.scrape_options)
+        scrape_data = prepare_scrape_options(request.scrape_options)
+        if scrape_data:
+            data["scrapeOptions"] = scrape_data
+        data.pop("scrape_options", None)
+    return data
+
+
+async def search(client: AsyncHttpClient, request: SearchRequest) -> SearchData:
+    payload = _prepare_search_request(request)
+    response = await client.post("/v2/search", payload)
+    if response.status_code >= 400:
+        handle_response_error(response, "search")
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+
+    data = body.get("data", {})
+    search_data = SearchData()
+    for source_type, source_documents in data.items():
+        if isinstance(source_documents, list):
+            results = []
+            for doc_data in source_documents:
+                if isinstance(doc_data, dict):
+                    if request.scrape_options is not None and any(
+                        key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']
+                    ):
+                        normalized = dict(doc_data)
+                        if 'rawHtml' in normalized and 'raw_html' not in normalized:
+                            normalized['raw_html'] = normalized.pop('rawHtml')
+                        if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+                            normalized['change_tracking'] = normalized.pop('changeTracking')
+                        results.append(Document(**normalized))
+                    else:
+                        results.append(SearchResult(
+                            url=doc_data.get('url', ''),
+                            title=doc_data.get('title'),
+                            description=doc_data.get('description')
+                        ))
+                elif isinstance(doc_data, str):
+                    results.append(SearchResult(url=doc_data))
+            if hasattr(search_data, source_type):
+                setattr(search_data, source_type, results)
+    return search_data
+

firecrawl/v2/methods/aio/usage.py

@@ -0,0 +1,42 @@
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.error_handler import handle_response_error
+from ...types import ConcurrencyCheck, CreditUsage, TokenUsage
+
+
+async def get_concurrency(client: AsyncHttpClient) -> ConcurrencyCheck:
+    resp = await client.get("/v2/concurrency-check")
+    if resp.status_code >= 400:
+        handle_response_error(resp, "get concurrency")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return ConcurrencyCheck(
+        concurrency=data.get("concurrency"),
+        max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
+    )
+
+
+async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage:
+    resp = await client.get("/v2/team/credit-usage")
+    if resp.status_code >= 400:
+        handle_response_error(resp, "get credit usage")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return CreditUsage(remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)))
+
+
+async def get_token_usage(client: AsyncHttpClient) -> TokenUsage:
+    resp = await client.get("/v2/team/token-usage")
+    if resp.status_code >= 400:
+        handle_response_error(resp, "get token usage")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return TokenUsage(
+        remaining_tokens=data.get("remainingTokens", 0)
+    )
+

firecrawl/v2/methods/batch.py

@@ -0,0 +1,420 @@
+"""
+Batch scraping functionality for Firecrawl v2 API.
+"""
+
+import time
+from typing import Optional, List, Callable, Dict, Any, Union
+from ..types import (
+    BatchScrapeRequest,
+    BatchScrapeResponse,
+    BatchScrapeJob,
+    ScrapeOptions,
+    Document,
+    WebhookConfig,
+)
+from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+from ..types import CrawlErrorsResponse
+
+
+def start_batch_scrape(
+    client: HttpClient,
+    urls: List[str],
+    *,
+    options: Optional[ScrapeOptions] = None,
+    webhook: Optional[Union[str, WebhookConfig]] = None,
+    append_to_id: Optional[str] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    max_concurrency: Optional[int] = None,
+    zero_data_retention: Optional[bool] = None,
+    integration: Optional[str] = None,
+    idempotency_key: Optional[str] = None,
+) -> BatchScrapeResponse:
+    """
+    Start a batch scrape job for multiple URLs.
+    
+    Args:
+        client: HTTP client instance
+        urls: List of URLs to scrape
+        options: Scraping options
+        
+    Returns:
+        BatchScrapeResponse containing job information
+        
+    Raises:
+        FirecrawlError: If the batch scrape operation fails to start
+    """
+    # Prepare request data
+    request_data = prepare_batch_scrape_request(
+        urls,
+        options=options,
+        webhook=webhook,
+        append_to_id=append_to_id,
+        ignore_invalid_urls=ignore_invalid_urls,
+        max_concurrency=max_concurrency,
+        zero_data_retention=zero_data_retention,
+        integration=integration,
+    )
+    
+    # Make the API request
+    headers = client._prepare_headers(idempotency_key)  # type: ignore[attr-defined]
+    response = client.post("/v2/batch/scrape", request_data, headers=headers)
+    
+    # Handle errors
+    if not response.ok:
+        handle_response_error(response, "start batch scrape")
+    
+    # Parse response
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+    return BatchScrapeResponse(
+        id=body.get("id"),
+        url=body.get("url"),
+        invalid_urls=body.get("invalidURLs") or None,
+    )
+
+
+def get_batch_scrape_status(
+    client: HttpClient,
+    job_id: str
+) -> BatchScrapeJob:
+    """
+    Get the status of a batch scrape job.
+    
+    Args:
+        client: HTTP client instance
+        job_id: ID of the batch scrape job
+        
+    Returns:
+        BatchScrapeStatusResponse containing job status and data
+        
+    Raises:
+        FirecrawlError: If the status check fails
+    """
+    # Make the API request
+    response = client.get(f"/v2/batch/scrape/{job_id}")
+    
+    # Handle errors
+    if not response.ok:
+        handle_response_error(response, "get batch scrape status")
+    
+    # Parse response
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+
+    # Convert documents
+    documents: List[Document] = []
+    for doc in body.get("data", []) or []:
+        if isinstance(doc, dict):
+            normalized = dict(doc)
+            if 'rawHtml' in normalized and 'raw_html' not in normalized:
+                normalized['raw_html'] = normalized.pop('rawHtml')
+            if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+                normalized['change_tracking'] = normalized.pop('changeTracking')
+            documents.append(Document(**normalized))
+
+    return BatchScrapeJob(
+        status=body.get("status"),
+        completed=body.get("completed", 0),
+        total=body.get("total", 0),
+        credits_used=body.get("creditsUsed"),
+        expires_at=body.get("expiresAt"),
+        next=body.get("next"),
+        data=documents,
+    )
+
+
+def cancel_batch_scrape(
+    client: HttpClient,
+    job_id: str
+) -> bool:
+    """
+    Cancel a running batch scrape job.
+    
+    Args:
+        client: HTTP client instance
+        job_id: ID of the batch scrape job to cancel
+        
+    Returns:
+        BatchScrapeStatusResponse with updated status
+        
+    Raises:
+        FirecrawlError: If the cancellation fails
+    """
+    # Make the API request
+    response = client.delete(f"/v2/batch/scrape/{job_id}")
+    
+    # Handle errors
+    if not response.ok:
+        handle_response_error(response, "cancel batch scrape")
+    
+    # Parse response
+    body = response.json()
+    return body.get("status") == "cancelled"
+
+
+def wait_for_batch_completion(
+    client: HttpClient,
+    job_id: str,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None
+) -> BatchScrapeJob:
+    """
+    Wait for a batch scrape job to complete, polling for status updates.
+    
+    Args:
+        client: HTTP client instance
+        job_id: ID of the batch scrape job
+        poll_interval: Seconds between status checks
+        timeout: Maximum seconds to wait (None for no timeout)
+        
+    Returns:
+        BatchScrapeStatusResponse when job completes
+        
+    Raises:
+        FirecrawlError: If the job fails or timeout is reached
+        TimeoutError: If timeout is reached
+    """
+    start_time = time.time()
+    
+    while True:
+        status_job = get_batch_scrape_status(client, job_id)
+        
+        # Check if job is complete
+        if status_job.status in ["completed", "failed", "cancelled"]:
+            return status_job
+        
+        # Check timeout
+        if timeout and (time.time() - start_time) > timeout:
+            raise TimeoutError(f"Batch scrape job {job_id} did not complete within {timeout} seconds")
+        
+        # Wait before next poll
+        time.sleep(poll_interval)
+
+
+def batch_scrape(
+    client: HttpClient,
+    urls: List[str],
+    *,
+    options: Optional[ScrapeOptions] = None,
+    webhook: Optional[Union[str, WebhookConfig]] = None,
+    append_to_id: Optional[str] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    max_concurrency: Optional[int] = None,
+    zero_data_retention: Optional[bool] = None,
+    integration: Optional[str] = None,
+    idempotency_key: Optional[str] = None,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None
+) -> BatchScrapeJob:
+    """
+    Start a batch scrape job and wait for it to complete.
+    
+    Args:
+        client: HTTP client instance
+        urls: List of URLs to scrape
+        options: Scraping options
+        poll_interval: Seconds between status checks
+        timeout: Maximum seconds to wait (None for no timeout)
+        
+    Returns:
+        BatchScrapeStatusResponse when job completes
+        
+    Raises:
+        FirecrawlError: If the batch scrape fails to start or complete
+        TimeoutError: If timeout is reached
+    """
+    # Start the batch scrape
+    start = start_batch_scrape(
+        client,
+        urls,
+        options=options,
+        webhook=webhook,
+        append_to_id=append_to_id,
+        ignore_invalid_urls=ignore_invalid_urls,
+        max_concurrency=max_concurrency,
+        zero_data_retention=zero_data_retention,
+        integration=integration,
+        idempotency_key=idempotency_key,
+    )
+
+    job_id = start.id
+
+    # Wait for completion
+    return wait_for_batch_completion(
+        client, job_id, poll_interval, timeout
+    )
+
+
+def validate_batch_urls(urls: List[str]) -> List[str]:
+    """
+    Validate and normalize a list of URLs for batch scraping.
+    
+    Args:
+        urls: List of URLs to validate
+        
+    Returns:
+        Validated list of URLs
+        
+    Raises:
+        ValueError: If URLs are invalid
+    """
+    if not urls:
+        raise ValueError("URLs list cannot be empty")
+    
+    if len(urls) > 1000:  # Assuming API limit
+        raise ValueError("Too many URLs (maximum 1000)")
+    
+    validated_urls = []
+    for url in urls:
+        if not url or not isinstance(url, str):
+            raise ValueError(f"Invalid URL: {url}")
+        
+        # Basic URL validation
+        if not (url.startswith("http://") or url.startswith("https://")):
+            raise ValueError(f"URL must start with http:// or https://: {url}")
+        
+        validated_urls.append(url.strip())
+    
+    return validated_urls
+
+
+def prepare_batch_scrape_request(
+    urls: List[str],
+    *,
+    options: Optional[ScrapeOptions] = None,
+    webhook: Optional[Union[str, WebhookConfig]] = None,
+    append_to_id: Optional[str] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    max_concurrency: Optional[int] = None,
+    zero_data_retention: Optional[bool] = None,
+    integration: Optional[str] = None,
+) -> dict:
+    """
+    Prepare a batch scrape request payload.
+    
+    Args:
+        urls: List of URLs to scrape
+        options: Scraping options
+        
+    Returns:
+        Request payload dictionary
+    """
+    validated_urls = validate_batch_urls(urls)
+    request_data: Dict[str, Any] = {"urls": validated_urls}
+
+    # Flatten scrape options at the top level (v2 behavior)
+    if options:
+        scrape_data = prepare_scrape_options(options)
+        if scrape_data:
+            request_data.update(scrape_data)
+
+    # Batch-specific fields
+    if webhook is not None:
+        if isinstance(webhook, str):
+            request_data["webhook"] = webhook
+        else:
+            request_data["webhook"] = webhook.model_dump(exclude_none=True)
+    if append_to_id is not None:
+        request_data["appendToId"] = append_to_id
+    if ignore_invalid_urls is not None:
+        request_data["ignoreInvalidURLs"] = ignore_invalid_urls
+    if max_concurrency is not None:
+        request_data["maxConcurrency"] = max_concurrency
+    if zero_data_retention is not None:
+        request_data["zeroDataRetention"] = zero_data_retention
+    if integration is not None:
+        request_data["integration"] = integration
+
+    return request_data
+
+
+def chunk_urls(urls: List[str], chunk_size: int = 100) -> List[List[str]]:
+    """
+    Split a large list of URLs into smaller chunks for batch processing.
+    
+    Args:
+        urls: List of URLs to chunk
+        chunk_size: Maximum size of each chunk
+        
+    Returns:
+        List of URL chunks
+    """
+    chunks = []
+    for i in range(0, len(urls), chunk_size):
+        chunks.append(urls[i:i + chunk_size])
+    return chunks
+
+
+def process_large_batch(
+    client: HttpClient,
+    urls: List[str],
+    options: Optional[ScrapeOptions] = None,
+    chunk_size: int = 100,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None
+) -> List[Document]:
+    """
+    Process a large batch of URLs by splitting into smaller chunks.
+    
+    Args:
+        client: HTTP client instance
+        urls: List of URLs to scrape
+        options: Scraping options
+        chunk_size: Size of each batch chunk
+        poll_interval: Seconds between status checks
+        timeout: Maximum seconds to wait per chunk
+        
+    Returns:
+        List of all scraped documents
+        
+    Raises:
+        FirecrawlError: If any chunk fails
+    """
+    url_chunks = chunk_urls(urls, chunk_size)
+    all_documents = []
+    completed_chunks = 0
+    
+    for chunk in url_chunks:
+        # Process this chunk
+        result = batch_scrape(
+            client,
+            chunk,
+            options=options,
+            poll_interval=poll_interval,
+            timeout=timeout,
+        )
+
+        # Add documents from this chunk
+        if result.data:
+            all_documents.extend(result.data)
+        
+        completed_chunks += 1
+    
+    return all_documents
+
+
+def get_batch_scrape_errors(client: HttpClient, job_id: str) -> CrawlErrorsResponse:
+    """
+    Get errors for a batch scrape job.
+
+    Args:
+        client: HTTP client instance
+        job_id: ID of the batch scrape job
+
+    Returns:
+        CrawlErrorsResponse with errors and robots-blocked URLs
+    """
+    response = client.get(f"/v2/batch/scrape/{job_id}/errors")
+
+    if not response.ok:
+        handle_response_error(response, "get batch scrape errors")
+
+    body = response.json()
+    payload = body.get("data", body)
+    normalized = {
+        "errors": payload.get("errors", []),
+        "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
+    }
+    return CrawlErrorsResponse(**normalized)