firecrawl 3.4.0__py3-none-any.whl → 4.0.0__py3-none-any.whl

firecrawl/v2/methods/crawl.py

@@ -3,12 +3,12 @@ Crawling functionality for Firecrawl v2 API.
 """
 
 import time
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 from ..types import (
     CrawlRequest,
     CrawlJob,
     CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData,
-    WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl
+    WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl, PaginationConfig
 )
 from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
 from ..utils.normalize import normalize_document_input
@@ -138,13 +138,18 @@ def start_crawl(client: HttpClient, request: CrawlRequest) -> CrawlResponse:
         raise Exception(response_data.get("error", "Unknown error occurred"))
 
 
-def get_crawl_status(client: HttpClient, job_id: str) -> CrawlJob:
+def get_crawl_status(
+    client: HttpClient, 
+    job_id: str,
+    pagination_config: Optional[PaginationConfig] = None
+) -> CrawlJob:
     """
     Get the status of a crawl job.
     
     Args:
         client: HTTP client instance
         job_id: ID of the crawl job
+        pagination_config: Optional configuration for pagination behavior
         
     Returns:
         CrawlJob with current status and data
@@ -176,6 +181,16 @@ def get_crawl_status(client: HttpClient, job_id: str) -> CrawlJob:
             else:
                 documents.append(Document(**normalize_document_input(doc_data)))
         
+        # Handle pagination if requested
+        auto_paginate = pagination_config.auto_paginate if pagination_config else True
+        if auto_paginate and response_data.get("next") and not (pagination_config and pagination_config.max_results is not None and len(documents) >= pagination_config.max_results):
+            documents = _fetch_all_pages(
+                client, 
+                response_data.get("next"), 
+                documents, 
+                pagination_config
+            )
+        
         # Create CrawlJob with current status and data
         return CrawlJob(
             status=response_data.get("status"),
@@ -183,13 +198,87 @@ def get_crawl_status(client: HttpClient, job_id: str) -> CrawlJob:
             total=response_data.get("total", 0),
             credits_used=response_data.get("creditsUsed", 0),
             expires_at=response_data.get("expiresAt"),
-            next=response_data.get("next", None),
+            next=response_data.get("next", None) if not auto_paginate else None,
             data=documents
         )
     else:
         raise Exception(response_data.get("error", "Unknown error occurred"))
 
 
+def _fetch_all_pages(
+    client: HttpClient,
+    next_url: str,
+    initial_documents: List[Document],
+    pagination_config: Optional[PaginationConfig] = None
+) -> List[Document]:
+    """
+    Fetch all pages of crawl results.
+    
+    Args:
+        client: HTTP client instance
+        next_url: URL for the next page
+        initial_documents: Documents from the first page
+        pagination_config: Optional configuration for pagination limits
+        
+    Returns:
+        List of all documents from all pages
+    """
+    documents = initial_documents.copy()
+    current_url = next_url
+    page_count = 0
+    
+    # Apply pagination limits
+    max_pages = pagination_config.max_pages if pagination_config else None
+    max_results = pagination_config.max_results if pagination_config else None
+    max_wait_time = pagination_config.max_wait_time if pagination_config else None
+    
+    start_time = time.monotonic()
+    
+    while current_url:
+        # Check pagination limits (treat 0 as a valid limit)
+        if (max_pages is not None) and page_count >= max_pages:
+            break
+
+        if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
+            break
+        
+        # Fetch next page
+        response = client.get(current_url)
+        
+        if not response.ok:
+            # Log error but continue with what we have
+            import logging
+            logger = logging.getLogger("firecrawl")
+            logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
+            break
+        
+        page_data = response.json()
+        
+        if not page_data.get("success"):
+            break
+        
+        # Add documents from this page
+        data_list = page_data.get("data", [])
+        for doc_data in data_list:
+            if isinstance(doc_data, str):
+                continue
+            else:
+                # Check max_results limit BEFORE adding each document
+                if max_results is not None and len(documents) >= max_results:
+                    break
+                documents.append(Document(**normalize_document_input(doc_data)))
+        
+        # Check if we hit max_results limit
+        if max_results is not None and len(documents) >= max_results:
+            break
+        
+        # Get next URL
+        current_url = page_data.get("next")
+        page_count += 1
+    
+    return documents
+
+
 def cancel_crawl(client: HttpClient, job_id: str) -> bool:
     """
     Cancel a running crawl job.
@@ -235,7 +324,7 @@ def wait_for_crawl_completion(
         Exception: If the job fails
         TimeoutError: If timeout is reached
     """
-    start_time = time.time()
+    start_time = time.monotonic()
     
     while True:
         crawl_job = get_crawl_status(client, job_id)
@@ -245,7 +334,7 @@ def wait_for_crawl_completion(
             return crawl_job
         
         # Check timeout
-        if timeout and (time.time() - start_time) > timeout:
+        if timeout is not None and (time.monotonic() - start_time) > timeout:
             raise TimeoutError(f"Crawl job {job_id} did not complete within {timeout} seconds")
         
         # Wait before next poll

firecrawl/v2/types.py

@@ -671,6 +671,13 @@ class ClientConfig(BaseModel):
     max_retries: int = 3
     backoff_factor: float = 0.5
 
+class PaginationConfig(BaseModel):
+    """Configuration for pagination behavior."""
+    auto_paginate: bool = True
+    max_pages: Optional[int] = Field(default=None, ge=0)
+    max_results: Optional[int] = Field(default=None, ge=0)
+    max_wait_time: Optional[int] = Field(default=None, ge=0)    # seconds
+
 # Response union types
 AnyResponse = Union[
     ScrapeResponse,

firecrawl/v2/utils/http_client.py

@@ -4,6 +4,7 @@ HTTP client utilities for v2 API.
 
 import time
 from typing import Dict, Any, Optional
+from urllib.parse import urlparse, urlunparse, urljoin
 import requests
 from .get_version import get_version
 
@@ -15,6 +16,28 @@ class HttpClient:
     def __init__(self, api_key: str, api_url: str):
         self.api_key = api_key
         self.api_url = api_url
+
+    def _build_url(self, endpoint: str) -> str:
+        base = urlparse(self.api_url)
+        ep = urlparse(endpoint)
+
+        # Absolute or protocol-relative (has netloc)
+        if ep.netloc:
+            # Different host: keep path/query but force base host/scheme (no token leakage)
+            path = ep.path or "/"
+            if (ep.hostname or "") != (base.hostname or ""):
+                return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
+            # Same host: normalize scheme to base
+            return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
+
+        # Relative (including leading slash or not)
+        base_str = self.api_url if self.api_url.endswith("/") else f"{self.api_url}/"
+        # Guard protocol-relative like //host/path slipping through as “relative”
+        if endpoint.startswith("//"):
+            ep2 = urlparse(f"https:{endpoint}")
+            path = ep2.path or "/"
+            return urlunparse((base.scheme or "https", base.netloc, path, "", ep2.query, ""))
+        return urljoin(base_str, endpoint)
     
     def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
         """Prepare headers for API requests."""
@@ -43,7 +66,7 @@ class HttpClient:
 
         data['origin'] = f'python-sdk@{version}'
             
-        url = f"{self.api_url}{endpoint}"
+        url = self._build_url(endpoint)
         
         last_exception = None
         
@@ -84,7 +107,7 @@ class HttpClient:
         if headers is None:
             headers = self._prepare_headers()
 
-        url = f"{self.api_url}{endpoint}"
+        url = self._build_url(endpoint)
         
         last_exception = None
         
@@ -124,7 +147,7 @@ class HttpClient:
         if headers is None:
             headers = self._prepare_headers()
             
-        url = f"{self.api_url}{endpoint}"
+        url = self._build_url(endpoint)
         
         last_exception = None
         

{firecrawl-3.4.0.dist-info → firecrawl-4.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: firecrawl
-Version: 3.4.0
+Version: 4.0.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/firecrawl/firecrawl
 Author: Mendable.ai

{firecrawl-3.4.0.dist-info → firecrawl-4.0.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
-firecrawl/__init__.py,sha256=Czdc4XJmhkDKcbSuq1wZsiWfezM8v9AUz0K_DEfOH-A,2192
+firecrawl/__init__.py,sha256=Fvnels2_mk87sESx-1GhfwBGCrRsN05RoUHsu4AswrY,2192
 firecrawl/client.py,sha256=tp3mUo_3aGPuZ53kpU4bhM-5EtwD_IUWrJ7wm0GMuCc,11159
 firecrawl/firecrawl.backup.py,sha256=v1FEN3jR4g5Aupg4xp6SLkuFvYMQuUKND2YELbYjE6c,200430
 firecrawl/types.py,sha256=W9N2pqQuevEIIjYHN9rbDf31E-nwdCECqIn11Foz2T8,2836
 firecrawl/__tests__/e2e/v2/conftest.py,sha256=I28TUpN5j0-9gM79NlbrDS8Jlsheao657od2f-2xK0Y,2587
 firecrawl/__tests__/e2e/v2/test_async.py,sha256=ZXpf1FVOJgNclITglrxIyFwP4cOiqzWLicGaxIm70BQ,2526
 firecrawl/__tests__/e2e/v2/test_batch_scrape.py,sha256=H9GtuwHIFdOQ958SOVThi_kvDDxcXAK_ECRh95ogonQ,3265
-firecrawl/__tests__/e2e/v2/test_crawl.py,sha256=cOssZvIwtghAtLiM1QdNLhPEwAxZ9j9umTrBUPtJjpU,9951
+firecrawl/__tests__/e2e/v2/test_crawl.py,sha256=fPbUZIKVtjENo8Mh4HyrwCVuqtNfn3Q1JrETufI27z4,9947
 firecrawl/__tests__/e2e/v2/test_extract.py,sha256=HgvGiDlyWtFygiPo5EP44Dem1oWrwgRF-hfc1LfeVSU,1670
 firecrawl/__tests__/e2e/v2/test_map.py,sha256=9sT-Yq8V_8c9esl_bv5hnTA9WXb2Dg81kj6M-s0484c,1618
 firecrawl/__tests__/e2e/v2/test_scrape.py,sha256=oyroF_WaEdxgD8t_SHkLBBfDRv1_6xZ_7vSTQpwlmA8,7198
@@ -25,6 +25,7 @@ firecrawl/__tests__/unit/v2/methods/test_crawl_params.py,sha256=p9hzg14uAs1iHKXP
 firecrawl/__tests__/unit/v2/methods/test_crawl_request_preparation.py,sha256=PEKbooNXfQwPpvcPHXABJnveztgAA-RFBhtlSs8uPro,8780
 firecrawl/__tests__/unit/v2/methods/test_crawl_validation.py,sha256=kErOmHSD01eMjXiMd4rgsMVGd_aU2G9uVymBjbAFoGw,3918
 firecrawl/__tests__/unit/v2/methods/test_map_request_preparation.py,sha256=toVcgnMp_cFeYsIUuyKGEWZGp0nAAkzaeFGUbY0zY0o,1868
+firecrawl/__tests__/unit/v2/methods/test_pagination.py,sha256=wNc9UtdauII_jzsjlJh645NBRq4IbQij1NeBwbyTjBU,22463
 firecrawl/__tests__/unit/v2/methods/test_scrape_request_preparation.py,sha256=wDOslsA5BN4kyezlaT5GeMv_Ifn8f461EaA7i5ujnaQ,3482
 firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py,sha256=14lUgFpQsiosgMKjDustBRVE0zXnHujBI76F8BC5PZ4,6072
 firecrawl/__tests__/unit/v2/methods/test_search_validation.py,sha256=7UGcNHpQzCpZbAPYjthfdPFWmAPcoApY-ED-khtuANs,9498
@@ -43,21 +44,21 @@ firecrawl/__tests__/unit/v2/watcher/test_ws_watcher.py,sha256=87w47n0iOihtu4jTR4
 firecrawl/v1/__init__.py,sha256=aP1oisPeZVGGZynvENc07JySMOZfv_4zAlxQ0ecMJXA,481
 firecrawl/v1/client.py,sha256=sydurfEFTsXyowyaGryA1lkPxN_r9Nf6iQpM43OwJyM,201672
 firecrawl/v2/__init__.py,sha256=Jc6a8tBjYG5OPkjDM5pl-notyys-7DEj7PLEfepv3fc,137
-firecrawl/v2/client.py,sha256=_DZFZO1aWvODzznK0g2Svcd2-xxXgWGR0d9vniNlk1w,30621
-firecrawl/v2/client_async.py,sha256=zwxHis1bSh0tSF1480ze-4XDQEDJ5yDur1ZqtL94dwc,10127
-firecrawl/v2/types.py,sha256=F-RCADQFdpAmF5t8LUabLOgyIV02Ol34yNa9y3S3ZMg,22667
+firecrawl/v2/client.py,sha256=aD4SDVKhh5glgzdnJ8JvCPCW_u8pv0BAfIg2Wffvjao,31137
+firecrawl/v2/client_async.py,sha256=XyzojIJlWatBGlAMish22H-XHkkH9zHsD6MGtAdtFg8,10487
+firecrawl/v2/types.py,sha256=RnFZf9CXBS3XkeB74L48sxWZ_ECZht4gVOKtyQLZz0o,22973
 firecrawl/v2/watcher.py,sha256=FOU71tqSKxgeuGycu4ye0SLc2dw7clIcoQjPsi-4Csc,14229
 firecrawl/v2/watcher_async.py,sha256=AVjW2mgABniolSsauK4u0FW8ya6WzRUdyEg2R-8vGCw,10278
-firecrawl/v2/methods/batch.py,sha256=us7zUGl7u9ZDIEk2J3rNqj87bkaNjXU27SMFW_fdcg8,11932
-firecrawl/v2/methods/crawl.py,sha256=4ZUmanHNuNtq9wbKMAZ3lenuPcNdOaV0kYXqMI5XJJ8,15485
+firecrawl/v2/methods/batch.py,sha256=jFSIPtvulUrPz3Y3zT1gDNwYEf8Botpfh4GOeYsVYRI,14852
+firecrawl/v2/methods/crawl.py,sha256=DWH1wUUDpE0zPSRALkQj_vF-PdsT0A1NyAGtnfcDaR8,18634
 firecrawl/v2/methods/extract.py,sha256=-Jr4BtraU3b7hd3JIY73V-S69rUclxyXyUpoQb6DCQk,4274
 firecrawl/v2/methods/map.py,sha256=4SADb0-lkbdOWDmO6k8_TzK0yRti5xsN40N45nUl9uA,2592
 firecrawl/v2/methods/scrape.py,sha256=CSHBwC-P91UfrW3zHirjNAs2h899FKcWvd1DY_4fJdo,1921
 firecrawl/v2/methods/search.py,sha256=6BKiQ1aKJjWBKm9BBtKxFKGD74kCKBeMIp_OgjcDFAw,7673
 firecrawl/v2/methods/usage.py,sha256=OJlkxwaB-AAtgO3WLr9QiqBRmjdh6GVhroCgleegupQ,1460
 firecrawl/v2/methods/aio/__init__.py,sha256=RocMJnGwnLIvGu3G8ZvY8INkipC7WHZiu2bE31eSyJs,35
-firecrawl/v2/methods/aio/batch.py,sha256=GS_xsd_Uib1fxFITBK1sH88VGzFMrIcqJVQqOvMQ540,3735
-firecrawl/v2/methods/aio/crawl.py,sha256=pC6bHVk30Hj1EJdAChxpMOg0Xx_GVqq4tIlvU2e5RQ4,6688
+firecrawl/v2/methods/aio/batch.py,sha256=4Uj05ffpMhQA2J_mkvHYYogdXb0IgbKGbomO43b4m94,6741
+firecrawl/v2/methods/aio/crawl.py,sha256=j2Tb2AcGsT6bCiUbB2yjrfvGZqkinUt0tU-SzWmB7Jw,11551
 firecrawl/v2/methods/aio/extract.py,sha256=IfNr2ETqt4dR73JFzrEYI4kk5vpKnJOG0BmPEjGEoO4,4217
 firecrawl/v2/methods/aio/map.py,sha256=EuT-5A0cQr_e5SBfEZ6pnl8u0JUwEEvSwhyT2N-QoKU,2326
 firecrawl/v2/methods/aio/scrape.py,sha256=ilA9qco8YGwCFpE0PN1XBQUyuHPQwH2QioZ-xsfxhgU,1386
@@ -66,14 +67,14 @@ firecrawl/v2/methods/aio/usage.py,sha256=OtBi6X-aT09MMR2dpm3vBCm9JrJZIJLCQ8jJ3L7
 firecrawl/v2/utils/__init__.py,sha256=i1GgxySmqEXpWSBQCu3iZBPIJG7fXj0QXCDWGwerWNs,338
 firecrawl/v2/utils/error_handler.py,sha256=Iuf916dHphDY8ObNNlWy75628DFeJ0Rv8ljRp4LttLE,4199
 firecrawl/v2/utils/get_version.py,sha256=0CxW_41q2hlzIxEWOivUCaYw3GFiSIH32RPUMcIgwAY,492
-firecrawl/v2/utils/http_client.py,sha256=_n8mp4xi6GGihg662Lsv6TSlvw9zykyADwEk0fg8mYA,4873
+firecrawl/v2/utils/http_client.py,sha256=gUrC1CvU5sj03w27Lbq-3-yH38Yi_OXiI01-piwA83w,6027
 firecrawl/v2/utils/http_client_async.py,sha256=iy89_bk2HS3afSRHZ8016eMCa9Fk-5MFTntcOHfbPgE,1936
 firecrawl/v2/utils/normalize.py,sha256=nlTU6QRghT1YKZzNZlIQj4STSRuSUGrS9cCErZIcY5w,3636
 firecrawl/v2/utils/validation.py,sha256=L8by7z-t6GuMGIYkK7il1BM8d-4_-sAdG9hDMF_LeG4,14518
-firecrawl-3.4.0.dist-info/licenses/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl-4.0.0.dist-info/licenses/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
 tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
 tests/test_timeout_conversion.py,sha256=PWlIEMASQNhu4cp1OW_ebklnE9NCiigPnEFCtI5N3w0,3996
-firecrawl-3.4.0.dist-info/METADATA,sha256=Gyv7wpufayVyAsLKfllRJihHmw43p5UH_lFDJf6i-mU,7392
-firecrawl-3.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-firecrawl-3.4.0.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
-firecrawl-3.4.0.dist-info/RECORD,,
+firecrawl-4.0.0.dist-info/METADATA,sha256=axsFmxpij8pHewgXTyRI3s5Rj96KoRG0Gxf4zCm11RY,7392
+firecrawl-4.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+firecrawl-4.0.0.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl-4.0.0.dist-info/RECORD,,