firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/v2/types.py

@@ -0,0 +1,546 @@
+"""
+Type definitions for Firecrawl v2 API.
+
+This module contains clean, modern type definitions for the v2 API.
+"""
+
+import warnings
+from datetime import datetime
+from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union
+from pydantic import BaseModel, Field, field_validator
+
+# Suppress pydantic warnings about schema field shadowing
+# Tested using schema_field alias="schema" but it doesn't work.
+warnings.filterwarnings("ignore", message="Field name \"schema\" in \"Format\" shadows an attribute in parent \"BaseModel\"")
+warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonFormat\" shadows an attribute in parent \"Format\"")
+warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingFormat\" shadows an attribute in parent \"Format\"")
+warnings.filterwarnings("ignore", message="Field name \"json\" in \"ScrapeFormats\" shadows an attribute in parent \"BaseModel\"")
+warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" shadows an attribute in parent \"BaseModel\"")
+
+T = TypeVar('T')
+
+# Base response types
+class BaseResponse(BaseModel, Generic[T]):
+    """Base response structure for all API responses."""
+    success: bool
+    data: Optional[T] = None
+    error: Optional[str] = None
+    warning: Optional[str] = None
+
+# Document and content types
+class DocumentMetadata(BaseModel):
+    """Metadata for scraped documents."""
+    title: Optional[str] = None
+    description: Optional[str] = None
+    language: Optional[str] = None
+    keywords: Optional[Union[str, List[str]]] = None
+    robots: Optional[str] = None
+    og_title: Optional[str] = None
+    og_description: Optional[str] = None
+    og_url: Optional[str] = None
+    og_image: Optional[str] = None
+    source_url: Optional[str] = None
+    status_code: Optional[int] = None
+    error: Optional[str] = None
+
+    @staticmethod
+    def _coerce_list_to_string(value: Any) -> Any:
+        if isinstance(value, list):
+            # Prefer first string if semantically a single-valued field, else join
+            if len(value) == 1:
+                return str(value[0])
+            return ', '.join(str(item) for item in value)
+        return value
+
+    @staticmethod
+    def _coerce_string_to_int(value: Any) -> Any:
+        if isinstance(value, str):
+            try:
+                return int(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_validator('robots', 'og_title', 'og_description', 'og_url', 'og_image', 'language', mode='before')
+    @classmethod
+    def coerce_lists_to_string_fields(cls, v):
+        return cls._coerce_list_to_string(v)
+
+    @field_validator('status_code', mode='before')
+    @classmethod
+    def coerce_status_code_to_int(cls, v):
+        return cls._coerce_string_to_int(v)
+
+class Document(BaseModel):
+    """A scraped document."""
+    markdown: Optional[str] = None
+    html: Optional[str] = None
+    raw_html: Optional[str] = None
+    json: Optional[Any] = None
+    summary: Optional[str] = None
+    metadata: Optional[DocumentMetadata] = None
+    links: Optional[List[str]] = None
+    screenshot: Optional[str] = None
+    actions: Optional[Dict[str, Any]] = None
+    warning: Optional[str] = None
+    change_tracking: Optional[Dict[str, Any]] = None
+
+# Webhook types
+class WebhookConfig(BaseModel):
+    """Configuration for webhooks."""
+    url: str
+    headers: Optional[Dict[str, str]] = None
+    metadata: Optional[Dict[str, str]] = None
+    events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None
+
+class WebhookData(BaseModel):
+    """Data sent to webhooks."""
+    job_id: str
+    status: str
+    current: Optional[int] = None
+    total: Optional[int] = None
+    data: Optional[List[Document]] = None
+    error: Optional[str] = None
+
+class Source(BaseModel):
+    """Configuration for a search source."""
+    type: str
+
+SourceOption = Union[str, Source]
+
+FormatString = Literal[
+    # camelCase versions (API format)
+    "markdown", "html", "rawHtml", "links", "screenshot", "summary", "changeTracking", "json",
+    # snake_case versions (user-friendly)
+    "raw_html", "change_tracking"
+]
+
+class Viewport(BaseModel):
+    """Viewport configuration for screenshots."""
+    width: int
+    height: int
+
+class Format(BaseModel):
+    """Configuration for a format."""
+    type: FormatString
+
+class JsonFormat(Format):
+    """Configuration for JSON extraction."""
+    prompt: Optional[str] = None
+    schema: Optional[Any] = None
+
+class ChangeTrackingFormat(Format):
+    """Configuration for change tracking."""
+    modes: List[Literal["git-diff", "json"]]
+    schema: Optional[Dict[str, Any]] = None
+    prompt: Optional[str] = None
+    tag: Optional[str] = None
+
+class ScreenshotFormat(BaseModel):
+    """Configuration for screenshot format."""
+    type: Literal["screenshot"] = "screenshot"
+    full_page: Optional[bool] = None
+    quality: Optional[int] = None
+    viewport: Optional[Union[Dict[str, int], Viewport]] = None
+
+FormatOption = Union[Dict[str, Any], FormatString, JsonFormat, ChangeTrackingFormat, ScreenshotFormat, Format]
+
+# Scrape types
+class ScrapeFormats(BaseModel):
+    """Output formats for scraping."""
+    formats: Optional[List[FormatOption]] = None
+    markdown: bool = True
+    html: bool = False
+    raw_html: bool = False
+    summary: bool = False
+    links: bool = False
+    screenshot: bool = False
+    change_tracking: bool = False
+    json: bool = False
+
+    @field_validator('formats')
+    @classmethod
+    def validate_formats(cls, v):
+        """Validate and normalize formats input."""
+        if v is None:
+            return v
+        
+        normalized_formats = []
+        for format_item in v:
+            if isinstance(format_item, str):
+                normalized_formats.append(Format(type=format_item))
+            elif isinstance(format_item, dict):
+                # Preserve dicts as-is to avoid dropping custom fields like 'schema'
+                normalized_formats.append(format_item)
+            elif isinstance(format_item, Format):
+                normalized_formats.append(format_item)
+            else:
+                raise ValueError(f"Invalid format format: {format_item}")
+        
+        return normalized_formats
+
+class ScrapeOptions(BaseModel):
+    """Options for scraping operations."""
+    formats: Optional[Union['ScrapeFormats', List[FormatOption]]] = None
+    headers: Optional[Dict[str, str]] = None
+    include_tags: Optional[List[str]] = None
+    exclude_tags: Optional[List[str]] = None
+    only_main_content: Optional[bool] = None
+    timeout: Optional[int] = None
+    wait_for: Optional[int] = None
+    mobile: Optional[bool] = None
+    parsers: Optional[List[str]] = None
+    actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None
+    location: Optional['Location'] = None
+    skip_tls_verification: Optional[bool] = None
+    remove_base64_images: Optional[bool] = None
+    fast_mode: Optional[bool] = None
+    use_mock: Optional[str] = None
+    block_ads: Optional[bool] = None
+    proxy: Optional[Literal["basic", "stealth", "auto"]] = None
+    max_age: Optional[int] = None
+    store_in_cache: Optional[bool] = None
+
+    @field_validator('formats')
+    @classmethod
+    def validate_formats(cls, v):
+        """Validate and normalize formats input."""
+        if v is None:
+            return v
+        if isinstance(v, ScrapeFormats):
+            return v
+        if isinstance(v, list):
+            return v
+        raise ValueError(f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]")
+
+class ScrapeRequest(BaseModel):
+    """Request for scraping a single URL."""
+    url: str
+    options: Optional[ScrapeOptions] = None
+
+class ScrapeData(Document):
+    """Scrape results data."""
+    pass
+
+class ScrapeResponse(BaseResponse[ScrapeData]):
+    """Response for scrape operations."""
+    pass
+
+# Crawl types
+class CrawlRequest(BaseModel):
+    """Request for crawling a website."""
+    url: str
+    prompt: Optional[str] = None
+    exclude_paths: Optional[List[str]] = None
+    include_paths: Optional[List[str]] = None
+    max_discovery_depth: Optional[int] = None
+    sitemap: Literal["skip", "include"] = "include"
+    ignore_query_parameters: bool = False
+    limit: Optional[int] = None
+    crawl_entire_domain: bool = False
+    allow_external_links: bool = False
+    allow_subdomains: bool = False
+    delay: Optional[int] = None
+    max_concurrency: Optional[int] = None
+    webhook: Optional[Union[str, WebhookConfig]] = None
+    scrape_options: Optional[ScrapeOptions] = None
+    zero_data_retention: bool = False
+
+class CrawlResponse(BaseModel):
+    """Information about a crawl job."""
+    id: str
+    url: str
+
+class CrawlJob(BaseModel):
+    """Crawl job status and progress data."""
+    status: Literal["scraping", "completed", "failed"]
+    total: int = 0
+    completed: int = 0
+    credits_used: int = 0
+    expires_at: Optional[datetime] = None
+    next: Optional[str] = None
+    data: List[Document] = []
+
+class SearchDocument(Document):
+    """A document from a search operation with URL and description."""
+    url: str
+    title: Optional[str] = None
+    description: Optional[str] = None
+
+class MapDocument(Document):
+    """A document from a map operation with URL and description."""
+    url: str
+    description: Optional[str] = None
+
+# Crawl params types
+class CrawlParamsRequest(BaseModel):
+    """Request for getting crawl parameters from LLM."""
+    url: str
+    prompt: str
+
+class CrawlParamsData(BaseModel):
+    """Data returned from crawl params endpoint."""
+    include_paths: Optional[List[str]] = None
+    exclude_paths: Optional[List[str]] = None
+    max_discovery_depth: Optional[int] = None
+    ignore_sitemap: bool = False
+    ignore_query_parameters: bool = False
+    limit: Optional[int] = None
+    crawl_entire_domain: bool = False
+    allow_external_links: bool = False
+    allow_subdomains: bool = False
+    delay: Optional[int] = None
+    max_concurrency: Optional[int] = None
+    webhook: Optional[Union[str, WebhookConfig]] = None
+    scrape_options: Optional[ScrapeOptions] = None
+    zero_data_retention: bool = False
+    warning: Optional[str] = None
+
+class CrawlParamsResponse(BaseResponse[CrawlParamsData]):
+    """Response from crawl params endpoint."""
+    pass
+
+# Batch scrape types
+class BatchScrapeRequest(BaseModel):
+    """Request for batch scraping multiple URLs (internal helper only)."""
+    urls: List[str]
+    options: Optional[ScrapeOptions] = None
+
+class BatchScrapeResponse(BaseModel):
+    """Response from starting a batch scrape job (mirrors CrawlResponse naming)."""
+    id: str
+    url: str
+    invalid_urls: Optional[List[str]] = None
+
+class BatchScrapeJob(BaseModel):
+    """Batch scrape job status and results."""
+    status: Literal["scraping", "completed", "failed", "cancelled"]
+    completed: int
+    total: int
+    credits_used: Optional[int] = None
+    expires_at: Optional[datetime] = None
+    next: Optional[str] = None
+    data: List[Document] = []
+
+# Map types
+class MapOptions(BaseModel):
+    """Options for mapping operations."""
+    search: Optional[str] = None
+    sitemap: Literal["only", "include", "skip"] = "include"
+    include_subdomains: Optional[bool] = None
+    limit: Optional[int] = None
+    timeout: Optional[int] = None
+
+class MapRequest(BaseModel):
+    """Request for mapping a website."""
+    url: str
+    options: Optional[MapOptions] = None
+
+class MapData(BaseModel):
+    """Map results data."""
+    links: List['SearchResult']
+
+class MapResponse(BaseResponse[MapData]):
+    """Response for map operations."""
+    pass
+
+# Extract types
+class ExtractResponse(BaseModel):
+    """Response for extract operations (start/status/final)."""
+    success: Optional[bool] = None
+    id: Optional[str] = None
+    status: Optional[Literal["processing", "completed", "failed", "cancelled"]] = None
+    data: Optional[Any] = None
+    error: Optional[str] = None
+    warning: Optional[str] = None
+    sources: Optional[Dict[str, Any]] = None
+    expires_at: Optional[datetime] = None
+
+# Usage/limits types
+class ConcurrencyCheck(BaseModel):
+    """Current concurrency and limits for the team/API key."""
+    concurrency: int
+    max_concurrency: int
+
+class CreditUsage(BaseModel):
+    """Remaining credits for the team/API key."""
+    remaining_credits: int
+
+class TokenUsage(BaseModel):
+    """Recent token usage metrics (if available)."""
+    remaining_tokens: int
+
+# Action types
+class WaitAction(BaseModel):
+    """Wait action to perform during scraping."""
+    type: Literal["wait"] = "wait"
+    milliseconds: Optional[int] = None
+    selector: Optional[str] = None
+
+class ScreenshotAction(BaseModel):
+    """Screenshot action to perform during scraping."""
+    type: Literal["screenshot"] = "screenshot"
+    full_page: Optional[bool] = None
+    quality: Optional[int] = None
+    viewport: Optional[Union[Dict[str, int], Viewport]] = None
+
+class ClickAction(BaseModel):
+    """Click action to perform during scraping."""
+    type: Literal["click"] = "click"
+    selector: str
+
+class WriteAction(BaseModel):
+    """Write action to perform during scraping."""
+    type: Literal["write"] = "write"
+    text: str
+
+class PressAction(BaseModel):
+    """Press action to perform during scraping."""
+    type: Literal["press"] = "press"
+    key: str
+
+class ScrollAction(BaseModel):
+    """Scroll action to perform during scraping."""
+    type: Literal["scroll"] = "scroll"
+    direction: Literal["up", "down"]
+    selector: Optional[str] = None
+
+class ScrapeAction(BaseModel):
+    """Scrape action to perform during scraping."""
+    type: Literal["scrape"] = "scrape"
+
+class ExecuteJavascriptAction(BaseModel):
+    """Execute javascript action to perform during scraping."""
+    type: Literal["executeJavascript"] = "executeJavascript"
+    script: str
+
+class PDFAction(BaseModel):
+    """PDF action to perform during scraping."""
+    type: Literal["pdf"] = "pdf"
+    format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None
+    landscape: Optional[bool] = None
+    scale: Optional[float] = None
+
+# Location types
+class Location(BaseModel):
+    """Location configuration for scraping."""
+    country: Optional[str] = None
+    languages: Optional[List[str]] = None
+
+class SearchRequest(BaseModel):
+    """Request for search operations."""
+    query: str
+    sources: Optional[List[SourceOption]] = None
+    limit: Optional[int] = 5
+    tbs: Optional[str] = None
+    location: Optional[str] = None
+    ignore_invalid_urls: Optional[bool] = None
+    timeout: Optional[int] = 60000
+    scrape_options: Optional[ScrapeOptions] = None
+
+    @field_validator('sources')
+    @classmethod
+    def validate_sources(cls, v):
+        """Validate and normalize sources input."""
+        if v is None:
+            return v
+        
+        normalized_sources = []
+        for source in v:
+            if isinstance(source, str):
+                normalized_sources.append(Source(type=source))
+            elif isinstance(source, dict):
+                normalized_sources.append(Source(**source))
+            elif isinstance(source, Source):
+                normalized_sources.append(source)
+            else:
+                raise ValueError(f"Invalid source format: {source}")
+        
+        return normalized_sources
+
+class LinkResult(BaseModel):
+    """A generic link result with optional metadata (used by search and map)."""
+    url: str
+    title: Optional[str] = None
+    description: Optional[str] = None
+    
+# Backward-compatible alias for existing tests/usages
+SearchResult = LinkResult
+
+class SearchData(BaseModel):
+    """Search results grouped by source type."""
+    web: Optional[List[Union[LinkResult, SearchDocument]]] = None
+    news: Optional[List[Union[LinkResult, SearchDocument]]] = None
+    images: Optional[List[Union[LinkResult, SearchDocument]]] = None
+
+class SearchResponse(BaseResponse[SearchData]):
+    """Response from search operation."""
+    pass
+
+# Error types
+class ErrorDetails(BaseModel):
+    """Detailed error information."""
+    code: Optional[str] = None
+    message: str
+    details: Optional[Dict[str, Any]] = None
+
+class ErrorResponse(BaseModel):
+    """Error response structure."""
+    success: bool = False
+    error: str
+    details: Optional[ErrorDetails] = None
+
+# Job management types
+class JobStatus(BaseModel):
+    """Generic job status information."""
+    id: str
+    status: Literal["pending", "scraping", "completed", "failed"]
+    current: Optional[int] = None
+    total: Optional[int] = None
+    created_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+    expires_at: Optional[datetime] = None
+
+class CrawlError(BaseModel):
+    """A crawl error."""
+    id: str
+    timestamp: Optional[datetime] = None
+    url: str
+    code: Optional[str] = None
+    error: str
+
+class CrawlErrorsResponse(BaseModel):
+    """Response from crawl error monitoring."""
+    errors: List[CrawlError]
+    robots_blocked: List[str]
+
+class ActiveCrawl(BaseModel):
+    """Information about an active crawl job."""
+    id: str
+    team_id: str
+    url: str
+    options: Optional[Dict[str, Any]] = None
+
+class ActiveCrawlsResponse(BaseModel):
+    """Response from active crawls endpoint."""
+    success: bool = True
+    crawls: List[ActiveCrawl]
+
+# Configuration types
+class ClientConfig(BaseModel):
+    """Configuration for the Firecrawl client."""
+    api_key: str
+    api_url: str = "https://api.firecrawl.dev"
+    timeout: Optional[float] = None
+    max_retries: int = 3
+    backoff_factor: float = 0.5
+
+# Response union types
+AnyResponse = Union[
+    ScrapeResponse,
+    CrawlResponse,
+    BatchScrapeResponse,
+    MapResponse,
+    SearchResponse,
+    ErrorResponse,
+]

firecrawl/v2/utils/__init__.py

@@ -0,0 +1,9 @@
+"""
+Utility modules for v2 API client.
+"""
+
+from .http_client import HttpClient
+from .error_handler import FirecrawlError, handle_response_error
+from .validation import validate_scrape_options, prepare_scrape_options
+
+__all__ = ['HttpClient', 'FirecrawlError', 'handle_response_error', 'validate_scrape_options', 'prepare_scrape_options']

firecrawl/v2/utils/error_handler.py

@@ -0,0 +1,107 @@
+"""
+Error handling utilities for v2 API.
+"""
+
+import requests
+from typing import Dict, Any, Optional
+
+
+class FirecrawlError(Exception):
+    """Base exception for Firecrawl API errors."""
+    
+    def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[requests.Response] = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.response = response
+
+
+class BadRequestError(FirecrawlError):
+    """Raised when the request is invalid (400)."""
+    pass
+
+
+
+class UnauthorizedError(FirecrawlError):
+    """Raised when the request is unauthorized (401)."""
+    pass
+
+
+class PaymentRequiredError(FirecrawlError):
+    """Raised when payment is required (402)."""
+    pass
+
+
+class WebsiteNotSupportedError(FirecrawlError):
+    """Raised when website is not supported (403)."""
+    pass
+
+
+class RequestTimeoutError(FirecrawlError):
+    """Raised when request times out (408)."""
+    pass
+
+
+class RateLimitError(FirecrawlError):
+    """Raised when the rate limit is exceeded (429)."""
+    pass
+
+
+class InternalServerError(FirecrawlError):
+    """Raised when there's an internal server error (500)."""
+    pass
+
+
+def handle_response_error(response: requests.Response, action: str) -> None:
+    """
+    Handle API response errors and raise appropriate exceptions.
+    
+    Args:
+        response: The HTTP response object
+        action: Description of the action being performed
+        
+    Raises:
+        FirecrawlError: Appropriate error based on status code
+    """
+    try:
+        response_json = response.json()
+        error_message = response_json.get('error', 'No error message provided.')
+        error_details = response_json.get('details', 'No additional error details provided.')
+    except:
+        # If we can't parse JSON, provide a helpful error message
+        try:
+            response_text = response.text[:500]  # Limit to first 500 chars
+            if response_text.strip():
+                error_message = f"Server returned non-JSON response: {response_text}"
+                error_details = f"Full response status: {response.status_code}"
+            else:
+                error_message = f"Server returned empty response with status {response.status_code}"
+                error_details = "No additional details available"
+        except:
+            error_message = f"Server returned unreadable response with status {response.status_code}"
+            error_details = "No additional details available"
+    
+    # Create appropriate error message
+    if response.status_code == 400:
+        message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"
+        raise BadRequestError(message, response.status_code, response)
+    elif response.status_code == 401:
+        message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"
+        raise UnauthorizedError(message, response.status_code, response)
+    elif response.status_code == 402:
+        message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
+        raise PaymentRequiredError(message, response.status_code, response)
+    elif response.status_code == 403:
+        message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+        raise WebsiteNotSupportedError(message, response.status_code, response)
+    elif response.status_code == 408:
+        message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
+        raise RequestTimeoutError(message, response.status_code, response)
+    elif response.status_code == 429:
+        message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"
+        raise RateLimitError(message, response.status_code, response)
+    elif response.status_code == 500:
+        message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
+        raise InternalServerError(message, response.status_code, response)
+    else:
+        message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message} - {error_details}"
+        raise FirecrawlError(message, response.status_code, response)

firecrawl/v2/utils/get_version.py

@@ -0,0 +1,15 @@
+import os
+import re
+from pathlib import Path
+
+def get_version():
+    try:
+        package_path = Path(__file__).parents[2]
+        version_file = (package_path / "__init__.py").read_text()
+        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+        if version_match:
+            return version_match.group(1).strip()
+        return "3.x.x"
+    except Exception as e:
+        print(f"Failed to get version from __init__.py: {e}")
+        return "3.x.x"