firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/client.py

@@ -0,0 +1,241 @@
+"""
+Firecrawl Client
+
+A Firecrawl client that enables you to scrape content from websites, crawl entire sites, search the web, and extract structured data using AI.
+
+The client supports both v1 and v2 API versions, providing access to features like:
+- Web scraping with advanced options (screenshots, markdown conversion, etc.)
+- Site crawling with configurable depth and limits
+- Web search with content extraction
+- Structured data extraction using AI models
+- Deep research capabilities
+
+Usage:
+    from firecrawl import Firecrawl
+    firecrawl = Firecrawl(api_key="your-api-key")
+    result = firecrawl.scrape("https://example.com")
+
+Check example.py for other usage examples.
+"""
+
+from typing import Any, Dict, Optional, List, Union
+import logging
+
+
+from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp
+from .v2 import FirecrawlClient as V2FirecrawlClient
+from .v2.client_async import AsyncFirecrawlClient
+
+logger = logging.getLogger("firecrawl")
+
+class V1Proxy:
+    """Type-annotated proxy for v1 client methods."""
+    _client: Optional[V1FirecrawlApp]
+    
+    def __init__(self, client_instance: Optional[V1FirecrawlApp]):
+        self._client = client_instance
+
+        if client_instance:
+            self.scrape_url = client_instance.scrape_url
+            self.crawl_url = client_instance.crawl_url
+            self.batch_scrape_urls = client_instance.batch_scrape_urls
+            self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls
+            self.async_crawl_url = client_instance.async_crawl_url
+            self.check_crawl_status = client_instance.check_crawl_status
+            self.map_url = client_instance.map_url
+            self.extract = client_instance.extract
+            self.deep_research = client_instance.deep_research
+            self.generate_llms_text = client_instance.generate_llms_text
+
+class V2Proxy:
+    """Proxy class that forwards method calls to the appropriate version client."""
+    _client: Optional[V2FirecrawlClient]
+    
+    def __init__(self, client_instance: Optional[V2FirecrawlClient]):
+        self._client = client_instance
+
+        if client_instance:
+            # self.scrape = client_instance.scrape
+            self.search = client_instance.search
+            self.crawl = client_instance.crawl
+            self.get_crawl_status = client_instance.get_crawl_status
+            self.cancel_crawl = client_instance.cancel_crawl
+            self.start_crawl = client_instance.start_crawl
+            self.crawl_params_preview = client_instance.crawl_params_preview
+            self.extract = client_instance.extract
+            self.start_batch_scrape = client_instance.start_batch_scrape
+            self.get_batch_scrape_status = client_instance.get_batch_scrape_status
+            self.cancel_batch_scrape = client_instance.cancel_batch_scrape
+            self.batch_scrape = client_instance.batch_scrape
+            self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
+            self.get_extract_status = client_instance.get_extract_status
+            self.map = client_instance.map
+            self.get_concurrency = client_instance.get_concurrency
+            self.get_credit_usage = client_instance.get_credit_usage
+            self.get_token_usage = client_instance.get_token_usage
+    
+    def __getattr__(self, name):
+        """Forward attribute access to the underlying client."""
+        return getattr(self._client, name)
+
+class AsyncV1Proxy:
+    """Type-annotated proxy for v1 client methods."""
+    _client: Optional[AsyncV1FirecrawlApp]
+    
+    def __init__(self, client_instance: Optional[AsyncV1FirecrawlApp]):
+        self._client = client_instance
+
+        if client_instance:
+            self.scrape_url = client_instance.scrape_url
+            self.crawl_url = client_instance.crawl_url
+            self.batch_scrape_urls = client_instance.batch_scrape_urls
+            self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls
+            self.async_crawl_url = client_instance.async_crawl_url
+            self.check_crawl_status = client_instance.check_crawl_status
+            self.map_url = client_instance.map_url
+            self.extract = client_instance.extract
+            self.deep_research = client_instance.deep_research
+            self.generate_llms_text = client_instance.generate_llms_text
+
+class AsyncV2Proxy:
+    """Proxy class that forwards method calls to the appropriate version client."""
+    _client: Optional[Any] = None
+
+    def __init__(self, client_instance: Optional[Any] = None):
+        self._client = client_instance
+
+        if client_instance:
+            self.scrape = client_instance.scrape
+            self.search = client_instance.search
+            self.crawl = client_instance.crawl
+            self.start_crawl = client_instance.start_crawl
+            self.wait_crawl = client_instance.wait_crawl
+            self.get_crawl_status = client_instance.get_crawl_status
+            self.cancel_crawl = client_instance.cancel_crawl
+            self.get_crawl_errors = client_instance.get_crawl_errors
+            self.get_active_crawls = client_instance.get_active_crawls
+            self.active_crawls = client_instance.active_crawls
+            self.crawl_params_preview = client_instance.crawl_params_preview
+
+            self.extract = client_instance.extract
+            self.start_extract = client_instance.start_extract
+            self.get_extract_status = client_instance.get_extract_status
+
+            self.start_batch_scrape = client_instance.start_batch_scrape
+            self.get_batch_scrape_status = client_instance.get_batch_scrape_status
+            self.cancel_batch_scrape = client_instance.cancel_batch_scrape
+            self.wait_batch_scrape = client_instance.wait_batch_scrape
+            self.batch_scrape = client_instance.batch_scrape
+            self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
+
+            self.map = client_instance.map
+            self.get_concurrency = client_instance.get_concurrency
+            self.get_credit_usage = client_instance.get_credit_usage
+            self.get_token_usage = client_instance.get_token_usage
+            self.watcher = client_instance.watcher
+
+    def __getattr__(self, name):
+        """Forward attribute access to the underlying client."""
+        if self._client:
+            return getattr(self._client, name)
+        raise AttributeError(f"Async v2 client not implemented yet: {name}")
+
+
+class Firecrawl:
+    """
+    Unified Firecrawl client (v2 by default, v1 under ``.v1``).
+
+    Provides a single entrypoint that exposes the latest API directly while
+    keeping a feature-frozen v1 available for incremental migration.
+    """
+    
+    def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+        """Initialize the unified client.
+
+        Args:
+            api_key: Firecrawl API key (or set ``FIRECRAWL_API_KEY``)
+            api_url: Base API URL (defaults to production)
+        """
+        self.api_key = api_key
+        self.api_url = api_url
+        
+        # Initialize version-specific clients
+        self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None
+        self._v2_client = V2FirecrawlClient(api_key=api_key, api_url=api_url) if V2FirecrawlClient else None
+        
+        # Create version-specific proxies
+        self.v1 = V1Proxy(self._v1_client) if self._v1_client else None
+        self.v2 = V2Proxy(self._v2_client)
+        
+        
+        self.scrape = self._v2_client.scrape
+        self.crawl = self._v2_client.crawl
+        self.start_crawl = self._v2_client.start_crawl
+        self.crawl_params_preview = self._v2_client.crawl_params_preview
+        self.get_crawl_status = self._v2_client.get_crawl_status
+        self.cancel_crawl = self._v2_client.cancel_crawl
+        self.get_crawl_errors = self._v2_client.get_crawl_errors
+        self.active_crawls = self._v2_client.active_crawls
+
+        self.start_batch_scrape = self._v2_client.start_batch_scrape
+        self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
+        self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
+        self.batch_scrape = self._v2_client.batch_scrape
+        self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
+        self.get_extract_status = self._v2_client.get_extract_status
+        self.map = self._v2_client.map
+        self.search = self._v2_client.search
+        self.extract = self._v2_client.extract
+        self.get_concurrency = self._v2_client.get_concurrency
+        self.get_credit_usage = self._v2_client.get_credit_usage
+        self.get_token_usage = self._v2_client.get_token_usage
+        self.watcher = self._v2_client.watcher
+        
+class AsyncFirecrawl:
+    """Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""
+
+    def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+        self.api_key = api_key
+        self.api_url = api_url
+        
+        # Initialize version-specific clients
+        self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None
+        self._v2_client = AsyncFirecrawlClient(api_key=api_key, api_url=api_url) if AsyncFirecrawlClient else None
+        
+        # Create version-specific proxies
+        self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None
+        self.v2 = AsyncV2Proxy(self._v2_client)
+
+        # Expose v2 async surface directly on the top-level client for ergonomic access
+        # Keep method names aligned with the sync client
+        self.scrape = self._v2_client.scrape
+        self.search = self._v2_client.search
+        self.map = self._v2_client.map
+
+        self.start_crawl = self._v2_client.start_crawl
+        self.get_crawl_status = self._v2_client.get_crawl_status
+        self.cancel_crawl = self._v2_client.cancel_crawl
+        self.crawl = self._v2_client.crawl
+        self.get_crawl_errors = self._v2_client.get_crawl_errors
+        self.active_crawls = self._v2_client.active_crawls
+        self.crawl_params_preview = self._v2_client.crawl_params_preview
+
+        self.start_batch_scrape = self._v2_client.start_batch_scrape
+        self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
+        self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
+        self.batch_scrape = self._v2_client.batch_scrape
+        self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
+
+        self.start_extract = self._v2_client.start_extract
+        self.get_extract_status = self._v2_client.get_extract_status
+        self.extract = self._v2_client.extract
+
+        self.get_concurrency = self._v2_client.get_concurrency
+        self.get_credit_usage = self._v2_client.get_credit_usage
+        self.get_token_usage = self._v2_client.get_token_usage
+
+        self.watcher = self._v2_client.watcher
+
+# Export Firecrawl as an alias for FirecrawlApp
+FirecrawlApp = Firecrawl
+AsyncFirecrawlApp = AsyncFirecrawl

firecrawl/{firecrawl.py → firecrawl.backup.py}

@@ -145,7 +145,7 @@ class ScrapeOptions(pydantic.BaseModel):
     excludeTags: Optional[List[str]] = None
     onlyMainContent: Optional[bool] = None
     waitFor: Optional[int] = None
-    timeout: Optional[int] = None
+    timeout: Optional[int] = 30000
     location: Optional[LocationConfig] = None
     mobile: Optional[bool] = None
     skipTlsVerification: Optional[bool] = None
@@ -258,6 +258,7 @@ class CrawlParams(pydantic.BaseModel):
     maxDiscoveryDepth: Optional[int] = None
     limit: Optional[int] = None
     allowBackwardLinks: Optional[bool] = None
+    crawlEntireDomain: Optional[bool] = None
     allowExternalLinks: Optional[bool] = None
     ignoreSitemap: Optional[bool] = None
     scrapeOptions: Optional[ScrapeOptions] = None
@@ -299,7 +300,7 @@ class MapParams(pydantic.BaseModel):
     includeSubdomains: Optional[bool] = None
     sitemapOnly: Optional[bool] = None
     limit: Optional[int] = None
-    timeout: Optional[int] = None
+    timeout: Optional[int] = 30000
     useIndex: Optional[bool] = None
 
 class MapResponse(pydantic.BaseModel):
@@ -463,7 +464,7 @@ class FirecrawlApp:
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
             wait_for: Optional[int] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             location: Optional[LocationConfig] = None,
             mobile: Optional[bool] = None,
             skip_tls_verification: Optional[bool] = None,
@@ -591,7 +592,7 @@ class FirecrawlApp:
             f'{self.api_url}/v1/scrape',
             headers=_headers,
             json=scrape_params,
-            timeout=(timeout + 5000 if timeout else None)
+            timeout=(timeout / 1000.0 + 5 if timeout is not None else None)
         )
 
         if response.status_code == 200:
@@ -618,7 +619,7 @@ class FirecrawlApp:
             lang: Optional[str] = None,
             country: Optional[str] = None,
             location: Optional[str] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             scrape_options: Optional[ScrapeOptions] = None,
             **kwargs) -> SearchResponse:
         """
@@ -1159,6 +1160,7 @@ class FirecrawlApp:
             max_discovery_depth=max_discovery_depth,
             limit=limit,
             allow_backward_links=allow_backward_links,
+            crawl_entire_domain=crawl_entire_domain,
             allow_external_links=allow_external_links,
             ignore_sitemap=ignore_sitemap,
             scrape_options=scrape_options,
@@ -1187,7 +1189,7 @@ class FirecrawlApp:
             include_subdomains: Optional[bool] = None,
             sitemap_only: Optional[bool] = None,
             limit: Optional[int] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             use_index: Optional[bool] = None,
             **kwargs) -> MapResponse:
         """
@@ -1278,7 +1280,7 @@ class FirecrawlApp:
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -1419,7 +1421,7 @@ class FirecrawlApp:
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -1559,7 +1561,7 @@ class FirecrawlApp:
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -2202,7 +2204,7 @@ class FirecrawlApp:
             requests.RequestException: If the request fails after the specified retries.
         """
         for attempt in range(retries):
-            response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] + 5000) if "timeout" in data else None))
+            response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] / 1000.0 + 5) if "timeout" in data and data["timeout"] is not None else None))
             if response.status_code == 502:
                 time.sleep(backoff_factor * (2 ** attempt))
             else:
@@ -2985,7 +2987,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
             wait_for: Optional[int] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             location: Optional[LocationConfig] = None,
             mobile: Optional[bool] = None,
             skip_tls_verification: Optional[bool] = None,
@@ -3123,7 +3125,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -3262,7 +3264,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -3747,7 +3749,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         include_subdomains: Optional[bool] = None,
         sitemap_only: Optional[bool] = None,
         limit: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         params: Optional[MapParams] = None) -> MapResponse:
         """
         Asynchronously map and discover links from a URL.
@@ -4481,7 +4483,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             lang: Optional[str] = None,
             country: Optional[str] = None,
             location: Optional[str] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             scrape_options: Optional[ScrapeOptions] = None,
             params: Optional[Union[Dict[str, Any], SearchParams]] = None,
             **kwargs) -> SearchResponse:

firecrawl/types.py

@@ -0,0 +1,157 @@
+"""
+Unified Firecrawl Types
+
+This module provides unified access to Firecrawl types across all API versions.
+Currently exports v2 types as the primary interface.
+"""
+
+from .v2.types import (
+    # Base types
+    BaseResponse,
+    
+    # Document types
+    Document,
+    DocumentMetadata,
+    
+    # Scrape types
+    ScrapeFormats,
+    ScrapeOptions,
+    ScrapeRequest,
+    ScrapeData,
+    ScrapeResponse,
+    
+    # Crawl types
+    CrawlRequest,
+    CrawlJob,
+    CrawlResponse,
+    CrawlParamsRequest,
+    CrawlParamsData,
+    CrawlParamsResponse,
+    CrawlErrorsResponse,
+    ActiveCrawlsResponse,
+    
+    # Batch scrape types
+    BatchScrapeRequest,
+    BatchScrapeJob,
+    BatchScrapeResponse,
+    
+    # Map types
+    MapOptions,
+    MapRequest,
+    MapData,
+    MapResponse,
+    
+    # Search types
+    Source,
+    SourceOption,
+    Format,
+    JsonFormat,
+    FormatOption,
+    SearchRequest,
+    SearchResult,
+    SearchData,
+    SearchResponse,
+    
+    # Action types
+    WaitAction,
+    ScreenshotAction,
+    ClickAction,
+    WriteAction,
+    PressAction,
+    ScrollAction,
+    ScrapeAction,
+    ExecuteJavascriptAction,
+    PDFAction,
+    
+    # Location and format types
+    Location,
+    
+    # Error types
+    ErrorDetails,
+    ErrorResponse,
+    
+    # Job management types
+    JobStatus,
+    
+    # Webhook types
+    WebhookData,
+    
+    # Configuration types
+    ClientConfig,
+)
+
+__all__ = [
+    # Base types
+    'BaseResponse',
+    
+    # Document types
+    'Document',
+    'DocumentMetadata',
+    
+    # Scrape types
+    'ScrapeFormats',
+    'ScrapeOptions',
+    'ScrapeRequest',
+    'ScrapeData',
+    'ScrapeResponse',
+    
+    # Crawl types
+    'CrawlRequest',
+    'CrawlJob',
+    'CrawlJobData',
+    'CrawlResponse',
+    'CrawlParamsRequest',
+    'CrawlParamsData',
+    'CrawlParamsResponse',
+    'CrawlErrorsResponse',
+    'ActiveCrawlsResponse',
+    
+    # Batch scrape types
+    'BatchScrapeRequest',
+    'BatchScrapeJob',
+    'BatchScrapeResponse',
+    
+    # Map types
+    'MapOptions',
+    'MapRequest',
+    'MapData',
+    'MapResponse',
+    
+    # Search types
+    'Source',
+    'SourceOption',
+    'Format',
+    'JsonFormat',
+    'FormatOption',
+    'SearchRequest',
+    'SearchResult',
+    'SearchData',
+    'SearchResponse',
+    
+    # Action types
+    'WaitAction',
+    'ScreenshotAction',
+    'ClickAction',
+    'WriteAction',
+    'PressAction',
+    'ScrollAction',
+    'ScrapeAction',
+    'ExecuteJavascriptAction',
+    'PDFAction',
+    
+    # Location and format types
+    'Location',
+    
+    # Error types
+    'ErrorDetails',
+    'ErrorResponse',
+    
+    # Job management types
+    'JobStatus',
+    
+    # Webhook types
+    'WebhookData',
+    
+    # Configuration types
+    'ClientConfig',
+] 

firecrawl/v1/__init__.py

@@ -0,0 +1,14 @@
+"""
+Firecrawl v1 API (Legacy)
+
+This module provides the legacy v1 API for backward compatibility.
+
+Usage:
+    from firecrawl.v1 import V1FirecrawlApp
+    app = V1FirecrawlApp(api_key="your-api-key")
+    result = app.scrape_url("https://example.com")
+"""
+
+from .client import V1FirecrawlApp, AsyncV1FirecrawlApp, V1JsonConfig, V1ScrapeOptions, V1ChangeTrackingOptions
+
+__all__ = ['V1FirecrawlApp', 'AsyncV1FirecrawlApp', 'V1JsonConfig', 'V1ScrapeOptions', 'V1ChangeTrackingOptions']