firecrawl 4.12.0__py3-none-any.whl

firecrawl/v2/utils/__init__.py

@@ -0,0 +1,9 @@
+"""
+Utility modules for v2 API client.
+"""
+
+from .http_client import HttpClient
+from .error_handler import FirecrawlError, handle_response_error
+from .validation import validate_scrape_options, prepare_scrape_options
+
+__all__ = ['HttpClient', 'FirecrawlError', 'handle_response_error', 'validate_scrape_options', 'prepare_scrape_options']

firecrawl/v2/utils/error_handler.py

@@ -0,0 +1,107 @@
+"""
+Error handling utilities for v2 API.
+"""
+
+import requests
+from typing import Dict, Any, Optional
+
+
+class FirecrawlError(Exception):
+    """Base exception for Firecrawl API errors."""
+    
+    def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[requests.Response] = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.response = response
+
+
+class BadRequestError(FirecrawlError):
+    """Raised when the request is invalid (400)."""
+    pass
+
+
+
+class UnauthorizedError(FirecrawlError):
+    """Raised when the request is unauthorized (401)."""
+    pass
+
+
+class PaymentRequiredError(FirecrawlError):
+    """Raised when payment is required (402)."""
+    pass
+
+
+class WebsiteNotSupportedError(FirecrawlError):
+    """Raised when website is not supported (403)."""
+    pass
+
+
+class RequestTimeoutError(FirecrawlError):
+    """Raised when request times out (408)."""
+    pass
+
+
+class RateLimitError(FirecrawlError):
+    """Raised when the rate limit is exceeded (429)."""
+    pass
+
+
+class InternalServerError(FirecrawlError):
+    """Raised when there's an internal server error (500)."""
+    pass
+
+
+def handle_response_error(response: requests.Response, action: str) -> None:
+    """
+    Handle API response errors and raise appropriate exceptions.
+    
+    Args:
+        response: The HTTP response object
+        action: Description of the action being performed
+        
+    Raises:
+        FirecrawlError: Appropriate error based on status code
+    """
+    try:
+        response_json = response.json()
+        error_message = response_json.get('error', 'No error message provided.')
+        error_details = response_json.get('details', 'No additional error details provided.')
+    except:
+        # If we can't parse JSON, provide a helpful error message
+        try:
+            response_text = response.text[:500]  # Limit to first 500 chars
+            if response_text.strip():
+                error_message = f"Server returned non-JSON response: {response_text}"
+                error_details = f"Full response status: {response.status_code}"
+            else:
+                error_message = f"Server returned empty response with status {response.status_code}"
+                error_details = "No additional details available"
+        except:
+            error_message = f"Server returned unreadable response with status {response.status_code}"
+            error_details = "No additional details available"
+    
+    # Create appropriate error message
+    if response.status_code == 400:
+        message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"
+        raise BadRequestError(message, response.status_code, response)
+    elif response.status_code == 401:
+        message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"
+        raise UnauthorizedError(message, response.status_code, response)
+    elif response.status_code == 402:
+        message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
+        raise PaymentRequiredError(message, response.status_code, response)
+    elif response.status_code == 403:
+        message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+        raise WebsiteNotSupportedError(message, response.status_code, response)
+    elif response.status_code == 408:
+        message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
+        raise RequestTimeoutError(message, response.status_code, response)
+    elif response.status_code == 429:
+        message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"
+        raise RateLimitError(message, response.status_code, response)
+    elif response.status_code == 500:
+        message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
+        raise InternalServerError(message, response.status_code, response)
+    else:
+        message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message} - {error_details}"
+        raise FirecrawlError(message, response.status_code, response)

firecrawl/v2/utils/get_version.py

@@ -0,0 +1,15 @@
+import os
+import re
+from pathlib import Path
+
+def get_version():
+    try:
+        package_path = Path(__file__).parents[2]
+        version_file = (package_path / "__init__.py").read_text()
+        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+        if version_match:
+            return version_match.group(1).strip()
+        return "3.x.x"
+    except Exception as e:
+        print(f"Failed to get version from __init__.py: {e}")
+        return "3.x.x"

firecrawl/v2/utils/http_client.py

@@ -0,0 +1,178 @@
+"""
+HTTP client utilities for v2 API.
+"""
+
+import time
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse, urlunparse, urljoin
+import requests
+from .get_version import get_version
+
+version = get_version()
+
+class HttpClient:
+    """HTTP client with retry logic and error handling."""
+
+    def __init__(self, api_key: Optional[str], api_url: str):
+        self.api_key = api_key
+        self.api_url = api_url
+
+    def _build_url(self, endpoint: str) -> str:
+        base = urlparse(self.api_url)
+        ep = urlparse(endpoint)
+
+        # Absolute or protocol-relative (has netloc)
+        if ep.netloc:
+            # Different host: keep path/query but force base host/scheme (no token leakage)
+            path = ep.path or "/"
+            if (ep.hostname or "") != (base.hostname or ""):
+                return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
+            # Same host: normalize scheme to base
+            return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
+
+        # Relative (including leading slash or not)
+        base_str = self.api_url if self.api_url.endswith("/") else f"{self.api_url}/"
+        # Guard protocol-relative like //host/path slipping through as “relative”
+        if endpoint.startswith("//"):
+            ep2 = urlparse(f"https:{endpoint}")
+            path = ep2.path or "/"
+            return urlunparse((base.scheme or "https", base.netloc, path, "", ep2.query, ""))
+        return urljoin(base_str, endpoint)
+    
+    def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
+        """Prepare headers for API requests."""
+        headers = {
+            'Content-Type': 'application/json',
+        }
+
+        if self.api_key:
+            headers['Authorization'] = f'Bearer {self.api_key}'
+        
+        if idempotency_key:
+            headers['x-idempotency-key'] = idempotency_key
+            
+        return headers
+    
+    def post(
+        self,
+        endpoint: str,
+        data: Dict[str, Any],
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+        retries: int = 3,
+        backoff_factor: float = 0.5
+    ) -> requests.Response:
+        """Make a POST request with retry logic."""
+        if headers is None:
+            headers = self._prepare_headers()
+
+        data['origin'] = f'python-sdk@{version}'
+            
+        url = self._build_url(endpoint)
+        
+        last_exception = None
+        
+        for attempt in range(retries):
+            try:
+                response = requests.post(
+                    url,
+                    headers=headers,
+                    json=data,
+                    timeout=timeout
+                )
+
+                if response.status_code == 502:
+                    if attempt < retries - 1:
+                        time.sleep(backoff_factor * (2 ** attempt))
+                        continue
+                
+                return response
+                
+            except requests.RequestException as e:
+                last_exception = e
+                if attempt == retries - 1:
+                    raise e
+                time.sleep(backoff_factor * (2 ** attempt))
+        
+        # This should never be reached due to the exception handling above
+        raise last_exception or Exception("Unexpected error in POST request")
+    
+    def get(
+        self,
+        endpoint: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+        retries: int = 3,
+        backoff_factor: float = 0.5
+    ) -> requests.Response:
+        """Make a GET request with retry logic."""
+        if headers is None:
+            headers = self._prepare_headers()
+
+        url = self._build_url(endpoint)
+        
+        last_exception = None
+        
+        for attempt in range(retries):
+            try:
+                response = requests.get(
+                    url,
+                    headers=headers,
+                    timeout=timeout
+                )
+                
+                if response.status_code == 502:
+                    if attempt < retries - 1:
+                        time.sleep(backoff_factor * (2 ** attempt))
+                        continue
+                
+                return response
+                
+            except requests.RequestException as e:
+                last_exception = e
+                if attempt == retries - 1:
+                    raise e
+                time.sleep(backoff_factor * (2 ** attempt))
+        
+        # This should never be reached due to the exception handling above
+        raise last_exception or Exception("Unexpected error in GET request")
+    
+    def delete(
+        self,
+        endpoint: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+        retries: int = 3,
+        backoff_factor: float = 0.5
+    ) -> requests.Response:
+        """Make a DELETE request with retry logic."""
+        if headers is None:
+            headers = self._prepare_headers()
+            
+        url = self._build_url(endpoint)
+        
+        last_exception = None
+        
+        for attempt in range(retries):
+            try:
+                response = requests.delete(
+                    url,
+                    headers=headers,
+                    timeout=timeout
+                )
+                
+                if response.status_code == 502:
+                    if attempt < retries - 1:
+                        time.sleep(backoff_factor * (2 ** attempt))
+                        continue
+                
+                return response
+                
+            except requests.RequestException as e:
+                last_exception = e
+                if attempt == retries - 1:
+                    raise e
+                time.sleep(backoff_factor * (2 ** attempt))
+        
+        # This should never be reached due to the exception handling above
+        raise last_exception or Exception("Unexpected error in DELETE request")

firecrawl/v2/utils/http_client_async.py

@@ -0,0 +1,69 @@
+import httpx
+from typing import Optional, Dict, Any
+from .get_version import get_version
+
+version = get_version()
+
+
+class AsyncHttpClient:
+    def __init__(self, api_key: Optional[str], api_url: str):
+        self.api_key = api_key
+        self.api_url = api_url
+        headers = {
+            "Content-Type": "application/json",
+        }
+
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+
+        self._client = httpx.AsyncClient(
+            base_url=api_url,
+            headers=headers,
+            limits=httpx.Limits(max_keepalive_connections=0),
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    def _headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
+        headers: Dict[str, str] = {}
+        if idempotency_key:
+            headers["x-idempotency-key"] = idempotency_key
+        return headers
+
+    async def post(
+        self,
+        endpoint: str,
+        data: Dict[str, Any],
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+    ) -> httpx.Response:
+        payload = dict(data)
+        payload["origin"] = f"python-sdk@{version}"
+        return await self._client.post(
+            endpoint,
+            json=payload,
+            headers={**self._headers(), **(headers or {})},
+            timeout=timeout,
+        )
+
+    async def get(
+        self,
+        endpoint: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+    ) -> httpx.Response:
+        return await self._client.get(
+            endpoint, headers={**self._headers(), **(headers or {})}, timeout=timeout
+        )
+
+    async def delete(
+        self,
+        endpoint: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+    ) -> httpx.Response:
+        return await self._client.delete(
+            endpoint, headers={**self._headers(), **(headers or {})}, timeout=timeout
+        )
+

firecrawl/v2/utils/normalize.py

@@ -0,0 +1,125 @@
+"""
+Normalization helpers for v2 API payloads to avoid relying on Pydantic aliases.
+"""
+
+from typing import Any, Dict, List
+from ..types import DocumentMetadata
+
+
+def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert API v2 camelCase metadata keys to snake_case expected by DocumentMetadata.
+    Leaves unknown keys as-is.
+    """
+    mapping = {
+        # OpenGraph
+        "ogTitle": "og_title",
+        "ogDescription": "og_description",
+        "ogUrl": "og_url",
+        "ogImage": "og_image",
+        "ogAudio": "og_audio",
+        "ogDeterminer": "og_determiner",
+        "ogLocale": "og_locale",
+        "ogLocaleAlternate": "og_locale_alternate",
+        "ogSiteName": "og_site_name",
+        "ogVideo": "og_video",
+        # Dublin Core and misc
+        "dcTermsCreated": "dc_terms_created",
+        "dcDateCreated": "dc_date_created",
+        "dcDate": "dc_date",
+        "dcTermsType": "dc_terms_type",
+        "dcType": "dc_type",
+        "dcTermsAudience": "dc_terms_audience",
+        "dcTermsSubject": "dc_terms_subject",
+        "dcSubject": "dc_subject",
+        "dcDescription": "dc_description",
+        "dcTermsKeywords": "dc_terms_keywords",
+        "modifiedTime": "modified_time",
+        "publishedTime": "published_time",
+        "articleTag": "article_tag",
+        "articleSection": "article_section",
+        # Response-level
+        "sourceURL": "source_url",
+        "statusCode": "status_code",
+        "scrapeId": "scrape_id",
+        "numPages": "num_pages",
+        "contentType": "content_type",
+        "proxyUsed": "proxy_used",
+        "cacheState": "cache_state",
+        "cachedAt": "cached_at",
+        "creditsUsed": "credits_used",
+        "concurrencyLimited": "concurrency_limited",
+        "concurrencyQueueDurationMs": "concurrency_queue_duration_ms",
+    }
+
+    out: Dict[str, Any] = {}
+    for k, v in md.items():
+        snake = mapping.get(k, k)
+        out[snake] = v
+
+    # Light coercions where server may send strings/lists
+    if isinstance(out.get("status_code"), str):
+        try:
+            out["status_code"] = int(out["status_code"])  # type: ignore
+        except ValueError:
+            pass
+
+    # Preserve list values for unknown keys; only lightweight coercions above
+    return out
+
+
+def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Normalize a raw Document dict from the API into the Python SDK's expected shape:
+    - Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking
+    - Convert metadata keys from camelCase to snake_case
+    - Convert branding.colorScheme to branding.color_scheme
+    """
+    normalized = dict(doc)
+
+    if "rawHtml" in normalized and "raw_html" not in normalized:
+        normalized["raw_html"] = normalized.pop("rawHtml")
+
+    if "changeTracking" in normalized and "change_tracking" not in normalized:
+        normalized["change_tracking"] = normalized.pop("changeTracking")
+
+    md = normalized.get("metadata")
+    if isinstance(md, dict):
+        mapped = _map_metadata_keys(md)
+        # Construct a typed DocumentMetadata; extras allowed/preserved
+        try:
+            normalized["metadata"] = DocumentMetadata.model_validate(mapped)
+        except Exception:
+            normalized["metadata"] = mapped
+
+    # Normalize branding top-level camelCase keys
+    branding = normalized.get("branding")
+    if isinstance(branding, dict):
+        if "colorScheme" in branding and "color_scheme" not in branding:
+            branding["color_scheme"] = branding.pop("colorScheme")
+
+    return normalized
+
+
+def _map_search_result_keys(result: Dict[str, Any], result_type: str) -> Dict[str, Any]:
+    if result_type == "images":
+        mapping = {
+            "imageUrl": "image_url",
+            "imageWidth": "image_width",
+            "imageHeight": "image_height",
+        }
+    elif result_type == "news":
+        mapping = {
+            "imageUrl": "image_url",
+        }
+    elif result_type == "web":
+        mapping = {}
+    else:
+        mapping = {}
+
+    out: Dict[str, Any] = {}
+    for k, v in result.items():
+        snake = mapping.get(k, k)
+        out[snake] = v
+
+    return out