firecrawl 4.12.0__py3-none-any.whl

firecrawl/v2/methods/extract.py

@@ -0,0 +1,161 @@
+from typing import Any, Dict, List, Optional
+import time
+
+from ..types import ExtractResponse, ScrapeOptions
+from ..types import AgentOptions
+from ..utils.http_client import HttpClient
+from ..utils.validation import prepare_scrape_options
+from ..utils.error_handler import handle_response_error
+
+
+def _prepare_extract_request(
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    integration: Optional[str] = None,
+    agent: Optional[AgentOptions] = None,
+) -> Dict[str, Any]:
+    body: Dict[str, Any] = {}
+    if urls is not None:
+        body["urls"] = urls
+    if prompt is not None:
+        body["prompt"] = prompt
+    if schema is not None:
+        body["schema"] = schema
+    if system_prompt is not None:
+        body["systemPrompt"] = system_prompt
+    if allow_external_links is not None:
+        body["allowExternalLinks"] = allow_external_links
+    if enable_web_search is not None:
+        body["enableWebSearch"] = enable_web_search
+    if show_sources is not None:
+        body["showSources"] = show_sources
+    if ignore_invalid_urls is not None:
+        body["ignoreInvalidURLs"] = ignore_invalid_urls
+    if scrape_options is not None:
+        prepared = prepare_scrape_options(scrape_options)
+        if prepared:
+            body["scrapeOptions"] = prepared
+    if integration is not None and str(integration).strip():
+        body["integration"] = str(integration).strip()
+    if agent is not None:
+        try:
+            body["agent"] = agent.model_dump(exclude_none=True)  # type: ignore[attr-defined]
+        except AttributeError:
+            body["agent"] = agent  # fallback
+    return body
+
+
+def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
+    out = dict(payload)
+    if "expiresAt" in out and "expires_at" not in out:
+        out["expires_at"] = out["expiresAt"]
+    if "creditsUsed" in out and "credits_used" not in out:
+        out["credits_used"] = out["creditsUsed"]
+    if "tokensUsed" in out and "tokens_used" not in out:
+        out["tokens_used"] = out["tokensUsed"]
+    return out
+
+
+def start_extract(
+    client: HttpClient,
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    integration: Optional[str] = None,
+    agent: Optional[AgentOptions] = None,
+) -> ExtractResponse:
+    body = _prepare_extract_request(
+        urls,
+        prompt=prompt,
+        schema=schema,
+        system_prompt=system_prompt,
+        allow_external_links=allow_external_links,
+        enable_web_search=enable_web_search,
+        show_sources=show_sources,
+        scrape_options=scrape_options,
+        ignore_invalid_urls=ignore_invalid_urls,
+        integration=integration,
+        agent=agent,
+    )
+    resp = client.post("/v2/extract", body)
+    if not resp.ok:
+        handle_response_error(resp, "extract")
+    payload = _normalize_extract_response_payload(resp.json())
+    return ExtractResponse(**payload)
+
+
+def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:
+    resp = client.get(f"/v2/extract/{job_id}")
+    if not resp.ok:
+        handle_response_error(resp, "extract-status")
+    payload = _normalize_extract_response_payload(resp.json())
+    return ExtractResponse(**payload)
+
+
+def wait_extract(
+    client: HttpClient,
+    job_id: str,
+    *,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None,
+) -> ExtractResponse:
+    start_ts = time.time()
+    while True:
+        status = get_extract_status(client, job_id)
+        if status.status in ("completed", "failed", "cancelled"):
+            return status
+        if timeout is not None and (time.time() - start_ts) > timeout:
+            return status
+        time.sleep(max(1, poll_interval))
+
+
+def extract(
+    client: HttpClient,
+    urls: Optional[List[str]],
+    *,
+    prompt: Optional[str] = None,
+    schema: Optional[Dict[str, Any]] = None,
+    system_prompt: Optional[str] = None,
+    allow_external_links: Optional[bool] = None,
+    enable_web_search: Optional[bool] = None,
+    show_sources: Optional[bool] = None,
+    scrape_options: Optional[ScrapeOptions] = None,
+    ignore_invalid_urls: Optional[bool] = None,
+    poll_interval: int = 2,
+    timeout: Optional[int] = None,
+    integration: Optional[str] = None,
+    agent: Optional[AgentOptions] = None,
+) -> ExtractResponse:
+    started = start_extract(
+        client,
+        urls,
+        prompt=prompt,
+        schema=schema,
+        system_prompt=system_prompt,
+        allow_external_links=allow_external_links,
+        enable_web_search=enable_web_search,
+        show_sources=show_sources,
+        scrape_options=scrape_options,
+        ignore_invalid_urls=ignore_invalid_urls,
+        integration=integration,
+        agent=agent,
+    )
+    job_id = getattr(started, "id", None)
+    if not job_id:
+        return started
+    return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)

firecrawl/v2/methods/map.py

@@ -0,0 +1,83 @@
+"""
+Mapping functionality for Firecrawl v2 API.
+"""
+
+from typing import Optional, Dict, Any
+from ..types import MapOptions, MapData, LinkResult
+from ..utils import HttpClient, handle_response_error
+
+
+def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
+    if not url or not url.strip():
+        raise ValueError("URL cannot be empty")
+
+    payload: Dict[str, Any] = {"url": url.strip()}
+
+    if options is not None:
+        # Unified sitemap parameter already provided in options
+        data: Dict[str, Any] = {}
+        if getattr(options, "sitemap", None) is not None:
+            data["sitemap"] = options.sitemap
+
+        if options.search is not None:
+            data["search"] = options.search
+        if options.include_subdomains is not None:
+            data["includeSubdomains"] = options.include_subdomains
+        if options.ignore_query_parameters is not None:
+            data["ignoreQueryParameters"] = options.ignore_query_parameters
+        if options.limit is not None:
+            data["limit"] = options.limit
+        if options.timeout is not None:
+            data["timeout"] = options.timeout
+        if options.integration is not None and options.integration.strip():
+            data["integration"] = options.integration.strip()
+        if options.location is not None:
+            data["location"] = options.location.model_dump(exclude_none=True)
+        payload.update(data)
+
+    return payload
+
+
+def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
+    """
+    Map a URL and return MapData (links list with optional titles/descriptions).
+    """
+    request_data = _prepare_map_request(url, options)
+    response = client.post("/v2/map", request_data)
+    if not response.ok:
+        handle_response_error(response, "map")
+
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+
+    # shouldnt return inside data?
+    # data = body.get("data", {})
+    # result_links: list[LinkResult] = []
+    # for item in data.get("links", []):
+    #     if isinstance(item, dict):
+    #         result_links.append(
+    #             LinkResult(
+    #                 url=item.get("url", ""),
+    #                 title=item.get("title"),
+    #                 description=item.get("description"),
+    #             )
+    #         )
+    #     elif isinstance(item, str):
+    #         result_links.append(LinkResult(url=item))
+
+    result_links: list[LinkResult] = []
+    for item in body.get("links", []):
+        if isinstance(item, dict):
+            result_links.append(
+                LinkResult(
+                    url=item.get("url", ""),
+                    title=item.get("title"),
+                    description=item.get("description"),
+                )
+            )
+        elif isinstance(item, str):
+            result_links.append(LinkResult(url=item))
+
+    return MapData(links=result_links)
+

firecrawl/v2/methods/scrape.py

@@ -0,0 +1,64 @@
+"""
+Scraping functionality for Firecrawl v2 API.
+"""
+
+from typing import Optional, Dict, Any
+from ..types import ScrapeOptions, Document
+from ..utils.normalize import normalize_document_input
+from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
+
+
+def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
+    """
+    Prepare a scrape request payload for v2 API.
+    
+    Args:
+        url: URL to scrape
+        options: ScrapeOptions (snake_case) to convert and include
+        
+    Returns:
+        Request payload dictionary with camelCase fields
+    """
+    if not url or not url.strip():
+        raise ValueError("URL cannot be empty")
+
+    request_data: Dict[str, Any] = {"url": url.strip()}
+
+    if options is not None:
+        validated = validate_scrape_options(options)
+        if validated is not None:
+            opts = prepare_scrape_options(validated)
+            if opts:
+                request_data.update(opts)
+
+    return request_data
+
+def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
+    """
+    Scrape a single URL and return the document.
+    
+    The v2 API returns: { success: boolean, data: Document }
+    We surface just the Document to callers.
+    
+    Args:
+        client: HTTP client instance
+        url: URL to scrape
+        options: Scraping options (snake_case)
+        
+    Returns:
+        Document
+    """
+    payload = _prepare_scrape_request(url, options)
+
+    response = client.post("/v2/scrape", payload)
+
+    if not response.ok:
+        handle_response_error(response, "scrape")
+
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+
+    document_data = body.get("data", {})
+    normalized = normalize_document_input(document_data)
+    return Document(**normalized)

firecrawl/v2/methods/search.py

@@ -0,0 +1,215 @@
+"""
+Search functionality for Firecrawl v2 API.
+"""
+
+import re
+from typing import Dict, Any, Union, List, TypeVar, Type
+from ..types import SearchRequest, SearchData, Document, SearchResultWeb, SearchResultNews, SearchResultImages
+from ..utils.normalize import normalize_document_input, _map_search_result_keys
+from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+
+T = TypeVar("T")
+
+def search(
+    client: HttpClient,
+    request: SearchRequest
+) -> SearchData:
+    """
+    Search for documents.
+    
+    Args:
+        client: HTTP client instance
+        request: Search request
+        
+    Returns:
+        SearchData with search results grouped by source type
+        
+    Raises:
+        FirecrawlError: If the search operation fails
+    """
+    request_data = _prepare_search_request(request)
+    try:
+        response = client.post("/v2/search", request_data)
+        if response.status_code != 200:
+            handle_response_error(response, "search")
+        response_data = response.json()
+        if not response_data.get("success"):
+            handle_response_error(response, "search")
+        data = response_data.get("data", {}) or {}
+        out = SearchData()
+        if "web" in data:
+            out.web = _transform_array(data["web"], SearchResultWeb)
+        if "news" in data:
+            out.news = _transform_array(data["news"], SearchResultNews)
+        if "images" in data:
+            out.images = _transform_array(data["images"], SearchResultImages)
+        return out
+    except Exception as err:
+        # If the error is an HTTP error from requests, handle it
+        # (simulate isAxiosError by checking for requests' HTTPError or Response)
+        if hasattr(err, "response"):
+            handle_response_error(getattr(err, "response"), "search")
+        raise err
+
+def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, 'Document']]:
+    """
+    Transforms an array of items into a list of result_type or Document.
+    If the item dict contains any of the special keys, it is treated as a Document.
+    Otherwise, it is treated as result_type.
+    If the item is not a dict, it is wrapped as result_type with url=item.
+    """
+    results: List[Union[T, 'Document']] = []
+    for item in arr:
+        if item and isinstance(item, dict):
+            if (
+                "markdown" in item or
+                "html" in item or
+                "rawHtml" in item or
+                "links" in item or
+                "screenshot" in item or
+                "changeTracking" in item or
+                "summary" in item or
+                "json" in item
+            ):
+                results.append(Document(**normalize_document_input(item)))
+            else:
+                result_type_name = None
+                if result_type == SearchResultImages:
+                    result_type_name = "images"
+                elif result_type == SearchResultNews:
+                    result_type_name = "news"
+                elif result_type == SearchResultWeb:
+                    result_type_name = "web"
+
+                if result_type_name:
+                    normalized_item = _map_search_result_keys(item, result_type_name)
+                    results.append(result_type(**normalized_item))
+                else:
+                    results.append(result_type(**item))
+        else:
+            results.append(result_type(url=item))
+    return results
+
+def _validate_search_request(request: SearchRequest) -> SearchRequest:
+    """
+    Validate and normalize search request.
+    
+    Args:
+        request: Search request to validate
+        
+    Returns:
+        Validated request
+        
+    Raises:
+        ValueError: If request is invalid
+    """
+    # Validate query
+    if not request.query or not request.query.strip():
+        raise ValueError("Query cannot be empty")
+    
+    # Validate limit
+    if request.limit is not None:
+        if request.limit <= 0:
+            raise ValueError("Limit must be positive")
+        if request.limit > 100:
+            raise ValueError("Limit cannot exceed 100")
+    
+    # Validate timeout
+    if request.timeout is not None:
+        if request.timeout <= 0:
+            raise ValueError("Timeout must be positive")
+        if request.timeout > 300000:  # 5 minutes max
+            raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
+    
+    # Validate sources (if provided)
+    if request.sources is not None:
+        valid_sources = {"web", "news", "images"}
+        for source in request.sources:
+            if isinstance(source, str):
+                if source not in valid_sources:
+                    raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
+            elif hasattr(source, 'type'):
+                if source.type not in valid_sources:
+                    raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
+    
+    # Validate categories (if provided)
+    if request.categories is not None:
+        valid_categories = {"github", "research", "pdf"}
+        for category in request.categories:
+            if isinstance(category, str):
+                if category not in valid_categories:
+                    raise ValueError(f"Invalid category type: {category}. Valid types: {valid_categories}")
+            elif hasattr(category, 'type'):
+                if category.type not in valid_categories:
+                    raise ValueError(f"Invalid category type: {category.type}. Valid types: {valid_categories}")
+    
+    # Validate location (if provided)
+    if request.location is not None:
+        if not isinstance(request.location, str) or len(request.location.strip()) == 0:
+            raise ValueError("Location must be a non-empty string")
+    
+    # Validate tbs (time-based search, if provided)
+    if request.tbs is not None:
+        valid_tbs_values = {
+            "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y",  # Google time filters
+            "d", "w", "m", "y"  # Short forms
+        }
+        
+        if request.tbs in valid_tbs_values:
+            pass  # Valid predefined value
+        elif request.tbs.startswith("cdr:"):
+            custom_date_pattern = r"^cdr:1,cd_min:\d{1,2}/\d{1,2}/\d{4},cd_max:\d{1,2}/\d{1,2}/\d{4}$"
+            if not re.match(custom_date_pattern, request.tbs):
+                raise ValueError(f"Invalid custom date range format: {request.tbs}. Expected format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
+        else:
+            raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values} or custom date range format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
+    
+    # Validate scrape_options (if provided)
+    if request.scrape_options is not None:
+        validate_scrape_options(request.scrape_options)
+    
+    return request
+
+
+def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
+    """
+    Prepare a search request payload.
+    
+    Args:
+        request: Search request
+        
+    Returns:
+        Request payload dictionary
+    """
+    validated_request = _validate_search_request(request)
+    data = validated_request.model_dump(exclude_none=True, by_alias=True)
+    
+    # Ensure default values are included only if not explicitly set to None
+    if "limit" not in data and validated_request.limit is not None:
+        data["limit"] = validated_request.limit
+    if "timeout" not in data and validated_request.timeout is not None:
+        data["timeout"] = validated_request.timeout
+    
+    # Handle snake_case to camelCase conversions manually
+    # (Pydantic Field() aliases interfere with value assignment)
+    
+    # ignore_invalid_urls → ignoreInvalidURLs
+    if validated_request.ignore_invalid_urls is not None:
+        data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
+        data.pop("ignore_invalid_urls", None)
+    
+    # scrape_options → scrapeOptions
+    if validated_request.scrape_options is not None:
+        scrape_data = prepare_scrape_options(validated_request.scrape_options)
+        if scrape_data:
+            data["scrapeOptions"] = scrape_data
+        data.pop("scrape_options", None)
+    
+    # Only include integration if it was explicitly provided and non-empty
+    integration_value = getattr(validated_request, "integration", None)
+    if integration_value is not None:
+        integration_str = str(integration_value).strip()
+        if integration_str:
+            data["integration"] = integration_str
+    
+    return data

firecrawl/v2/methods/usage.py

@@ -0,0 +1,84 @@
+from ..utils import HttpClient, handle_response_error
+from ..types import ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsage, CreditUsageHistoricalResponse, TokenUsageHistoricalResponse
+
+
+def get_concurrency(client: HttpClient) -> ConcurrencyCheck:
+    resp = client.get("/v2/concurrency-check")
+    if not resp.ok:
+        handle_response_error(resp, "get concurrency")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return ConcurrencyCheck(
+        concurrency=data.get("concurrency"),
+        max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
+    )
+
+
+def get_credit_usage(client: HttpClient) -> CreditUsage:
+    resp = client.get("/v2/team/credit-usage")
+    if not resp.ok:
+        handle_response_error(resp, "get credit usage")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return CreditUsage(
+        remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)),
+        plan_credits=data.get("planCredits", data.get("plan_credits")),
+        billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
+        billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
+    )
+
+
+def get_token_usage(client: HttpClient) -> TokenUsage:
+    resp = client.get("/v2/team/token-usage")
+    if not resp.ok:
+        handle_response_error(resp, "get token usage")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return TokenUsage(
+        remaining_tokens=data.get("remainingTokens", data.get("remaining_tokens", 0)),
+        plan_tokens=data.get("planTokens", data.get("plan_tokens")),
+        billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
+        billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
+    )
+
+def get_queue_status(client: HttpClient) -> QueueStatusResponse:
+    resp = client.get("/v2/team/queue-status")
+    if not resp.ok:
+        handle_response_error(resp, "get queue status")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return QueueStatusResponse(
+        jobs_in_queue=data.get("jobsInQueue", 0),
+        active_jobs_in_queue=data.get("activeJobsInQueue", 0),
+        waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),
+        max_concurrency=data.get("maxConcurrency", 0),
+        most_recent_success=data.get("mostRecentSuccess", None),
+    )
+
+
+def get_credit_usage_historical(client: HttpClient, by_api_key: bool = False) -> CreditUsageHistoricalResponse:
+    resp = client.get(f"/v2/team/credit-usage/historical{'?byApiKey=true' if by_api_key else ''}")
+    if not resp.ok:
+        handle_response_error(resp, "get credit usage historical")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    return CreditUsageHistoricalResponse(**body)
+
+
+def get_token_usage_historical(client: HttpClient, by_api_key: bool = False) -> TokenUsageHistoricalResponse:
+    resp = client.get(f"/v2/team/token-usage/historical{'?byApiKey=true' if by_api_key else ''}")
+    if not resp.ok:
+        handle_response_error(resp, "get token usage historical")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    return TokenUsageHistoricalResponse(**body)