firecrawl-py 3.0.3__py3-none-any.whl → 3.1.1__py3-none-any.whl

firecrawl/__init__.py

@@ -17,7 +17,7 @@ from .v1 import (
     V1ChangeTrackingOptions,
 )
 
-__version__ = "3.0.3"
+__version__ = "3.1.1"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")
@@ -84,4 +84,4 @@ __all__ = [
     'V1JsonConfig',
     'V1ScrapeOptions',
     'V1ChangeTrackingOptions',
-]
+]

firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py

@@ -96,7 +96,6 @@ async def test_async_get_crawl_status_shape():
     assert status.status in ("scraping", "completed", "failed")
     assert status.completed >= 0
     assert status.expires_at is not None
-    assert status.next is not None
     assert isinstance(status.data, list)
 
 

firecrawl/__tests__/unit/v2/methods/test_search_validation.py

@@ -11,7 +11,7 @@ class TestSearchValidation:
         request = SearchRequest(query="")
         with pytest.raises(ValueError, match="Query cannot be empty"):
             _validate_search_request(request)
-        
+
         request = SearchRequest(query="   ")
         with pytest.raises(ValueError, match="Query cannot be empty"):
             _validate_search_request(request)
@@ -22,12 +22,12 @@ class TestSearchValidation:
         request = SearchRequest(query="test", limit=0)
         with pytest.raises(ValueError, match="Limit must be positive"):
             _validate_search_request(request)
-        
+
         # Negative limit
         request = SearchRequest(query="test", limit=-1)
         with pytest.raises(ValueError, match="Limit must be positive"):
             _validate_search_request(request)
-        
+
         # Too high limit
         request = SearchRequest(query="test", limit=101)
         with pytest.raises(ValueError, match="Limit cannot exceed 100"):
@@ -39,12 +39,12 @@ class TestSearchValidation:
         request = SearchRequest(query="test", timeout=0)
         with pytest.raises(ValueError, match="Timeout must be positive"):
             _validate_search_request(request)
-        
+
         # Negative timeout
         request = SearchRequest(query="test", timeout=-1000)
         with pytest.raises(ValueError, match="Timeout must be positive"):
             _validate_search_request(request)
-        
+
         # Too high timeout
         request = SearchRequest(query="test", timeout=300001)
         with pytest.raises(ValueError, match="Timeout cannot exceed 300000ms"):
@@ -56,12 +56,12 @@ class TestSearchValidation:
         request = SearchRequest(query="test", sources=["invalid_source"])
         with pytest.raises(ValueError, match="Invalid source type"):
             _validate_search_request(request)
-        
+
         # Invalid object source
         request = SearchRequest(query="test", sources=[Source(type="invalid_source")])
         with pytest.raises(ValueError, match="Invalid source type"):
             _validate_search_request(request)
-        
+
         # Mixed valid/invalid sources
         request = SearchRequest(query="test", sources=["web", "invalid_source"])
         with pytest.raises(ValueError, match="Invalid source type"):
@@ -73,7 +73,7 @@ class TestSearchValidation:
         request = SearchRequest(query="test", location="")
         with pytest.raises(ValueError, match="Location must be a non-empty string"):
             _validate_search_request(request)
-        
+
         # Whitespace location
         request = SearchRequest(query="test", location="   ")
         with pytest.raises(ValueError, match="Location must be a non-empty string"):
@@ -82,19 +82,49 @@ class TestSearchValidation:
     def test_validate_invalid_tbs(self):
         """Test validation of invalid tbs values."""
         invalid_tbs_values = ["invalid", "qdr:x", "yesterday", "last_week"]
-        
+
         for invalid_tbs in invalid_tbs_values:
             request = SearchRequest(query="test", tbs=invalid_tbs)
             with pytest.raises(ValueError, match="Invalid tbs value"):
                 _validate_search_request(request)
 
+    def test_validate_custom_date_ranges(self):
+        """Test validation of custom date range formats."""
+        valid_custom_ranges = [
+            "cdr:1,cd_min:1/1/2024,cd_max:12/31/2024",
+            "cdr:1,cd_min:12/1/2024,cd_max:12/31/2024",
+            "cdr:1,cd_min:2/28/2023,cd_max:3/1/2023",
+            "cdr:1,cd_min:10/15/2023,cd_max:11/15/2023"
+        ]
+
+        for valid_range in valid_custom_ranges:
+            request = SearchRequest(query="test", tbs=valid_range)
+            validated = _validate_search_request(request)
+            assert validated == request
+
+    def test_validate_invalid_custom_date_ranges(self):
+        """Test validation of invalid custom date range formats."""
+        # Invalid custom date ranges
+        invalid_custom_ranges = [
+            "cdr:1,cd_min:2/28/2023",  # Missing cd_max
+            "cdr:1,cd_max:2/28/2023",  # Missing cd_min
+            "cdr:2,cd_min:1/1/2024,cd_max:12/31/2024",  # Wrong cdr value
+            "cdr:cd_min:1/1/2024,cd_max:12/31/2024",  # Missing :1
+            "custom:1,cd_min:1/1/2024,cd_max:12/31/2024"  # Wrong prefix
+        ]
+
+        for invalid_range in invalid_custom_ranges:
+            request = SearchRequest(query="test", tbs=invalid_range)
+            with pytest.raises(ValueError, match="Invalid"):
+                _validate_search_request(request)
+
     def test_validate_valid_requests(self):
         """Test that valid requests pass validation."""
         # Minimal valid request
         request = SearchRequest(query="test")
         validated = _validate_search_request(request)
         assert validated == request
-        
+
         # Request with all optional parameters
         request = SearchRequest(
             query="test query",
@@ -107,7 +137,7 @@ class TestSearchValidation:
         )
         validated = _validate_search_request(request)
         assert validated == request
-        
+
         # Request with object sources
         request = SearchRequest(
             query="test",
@@ -122,17 +152,17 @@ class TestSearchValidation:
         request = SearchRequest(query="test", limit=100)
         validated = _validate_search_request(request)
         assert validated == request
-        
+
         # Maximum valid timeout
         request = SearchRequest(query="test", timeout=300000)
         validated = _validate_search_request(request)
         assert validated == request
-        
+
         # Minimum valid limit
         request = SearchRequest(query="test", limit=1)
         validated = _validate_search_request(request)
         assert validated == request
-        
+
         # Minimum valid timeout
         request = SearchRequest(query="test", timeout=1)
         validated = _validate_search_request(request)
@@ -191,16 +221,16 @@ class TestSearchRequestModel:
         data1 = request1.model_dump(by_alias=True)
         assert "ignore_invalid_urls" in data1  # No alias, uses snake_case
         assert data1["ignore_invalid_urls"] is None
-        
+
         # Test with explicit False value
         request2 = SearchRequest(
             query="test",
             ignore_invalid_urls=False,
             scrape_options=ScrapeOptions(formats=["markdown"])
         )
-        
+
         # Check that aliases are used in model_dump with by_alias=True
         data2 = request2.model_dump(by_alias=True)
         assert "ignore_invalid_urls" in data2  # No alias, uses snake_case
         assert "scrape_options" in data2  # No alias, uses snake_case
-        assert data2["ignore_invalid_urls"] is False 
+        assert data2["ignore_invalid_urls"] is False

firecrawl/client.py

@@ -25,6 +25,7 @@ import logging
 from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp
 from .v2 import FirecrawlClient as V2FirecrawlClient
 from .v2.client_async import AsyncFirecrawlClient
+from .v2.types import Document
 
 logger = logging.getLogger("firecrawl")
 

firecrawl/v2/methods/aio/crawl.py

@@ -14,6 +14,7 @@ from ...types import (
 from ...utils.error_handler import handle_response_error
 from ...utils.validation import prepare_scrape_options
 from ...utils.http_client_async import AsyncHttpClient
+from ...utils.normalize import normalize_document_input
 
 
 def _prepare_crawl_request(request: CrawlRequest) -> dict:
@@ -76,11 +77,7 @@ async def get_crawl_status(client: AsyncHttpClient, job_id: str) -> CrawlJob:
         documents = []
         for doc_data in body.get("data", []):
             if isinstance(doc_data, dict):
-                normalized = dict(doc_data)
-                if 'rawHtml' in normalized and 'raw_html' not in normalized:
-                    normalized['raw_html'] = normalized.pop('rawHtml')
-                if 'changeTracking' in normalized and 'change_tracking' not in normalized:
-                    normalized['change_tracking'] = normalized.pop('changeTracking')
+                normalized = normalize_document_input(doc_data)
                 documents.append(Document(**normalized))
         return CrawlJob(
             status=body.get("status"),

firecrawl/v2/methods/aio/scrape.py

@@ -1,5 +1,6 @@
 from typing import Optional, Dict, Any
 from ...types import ScrapeOptions, Document
+from ...utils.normalize import normalize_document_input
 from ...utils.error_handler import handle_response_error
 from ...utils.validation import prepare_scrape_options, validate_scrape_options
 from ...utils.http_client_async import AsyncHttpClient
@@ -27,10 +28,6 @@ async def scrape(client: AsyncHttpClient, url: str, options: Optional[ScrapeOpti
     if not body.get("success"):
         raise Exception(body.get("error", "Unknown error occurred"))
     document_data = body.get("data", {})
-    normalized = dict(document_data)
-    if 'rawHtml' in normalized and 'raw_html' not in normalized:
-        normalized['raw_html'] = normalized.pop('rawHtml')
-    if 'changeTracking' in normalized and 'change_tracking' not in normalized:
-        normalized['change_tracking'] = normalized.pop('changeTracking')
+    normalized = normalize_document_input(document_data)
     return Document(**normalized)
 

firecrawl/v2/methods/aio/search.py

@@ -1,5 +1,6 @@
 from typing import Dict, Any
 from ...types import SearchRequest, SearchData, SearchResult, Document
+from ...utils.normalize import normalize_document_input
 from ...utils.http_client_async import AsyncHttpClient
 from ...utils.error_handler import handle_response_error
 from ...utils.validation import prepare_scrape_options, validate_scrape_options
@@ -38,11 +39,7 @@ async def search(client: AsyncHttpClient, request: SearchRequest) -> SearchData:
                     if request.scrape_options is not None and any(
                         key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']
                     ):
-                        normalized = dict(doc_data)
-                        if 'rawHtml' in normalized and 'raw_html' not in normalized:
-                            normalized['raw_html'] = normalized.pop('rawHtml')
-                        if 'changeTracking' in normalized and 'change_tracking' not in normalized:
-                            normalized['change_tracking'] = normalized.pop('changeTracking')
+                        normalized = normalize_document_input(doc_data)
                         results.append(Document(**normalized))
                     else:
                         results.append(SearchResult(

firecrawl/v2/methods/batch.py

@@ -13,6 +13,7 @@ from ..types import (
     WebhookConfig,
 )
 from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+from ..utils.normalize import normalize_document_input
 from ..types import CrawlErrorsResponse
 
 
@@ -107,11 +108,7 @@ def get_batch_scrape_status(
     documents: List[Document] = []
     for doc in body.get("data", []) or []:
         if isinstance(doc, dict):
-            normalized = dict(doc)
-            if 'rawHtml' in normalized and 'raw_html' not in normalized:
-                normalized['raw_html'] = normalized.pop('rawHtml')
-            if 'changeTracking' in normalized and 'change_tracking' not in normalized:
-                normalized['change_tracking'] = normalized.pop('changeTracking')
+            normalized = normalize_document_input(doc)
             documents.append(Document(**normalized))
 
     return BatchScrapeJob(

firecrawl/v2/methods/crawl.py

@@ -11,6 +11,7 @@ from ..types import (
     WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl
 )
 from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+from ..utils.normalize import normalize_document_input
 
 
 def _validate_crawl_request(request: CrawlRequest) -> None:
@@ -173,7 +174,7 @@ def get_crawl_status(client: HttpClient, job_id: str) -> CrawlJob:
                 # but we'll handle it gracefully
                 continue
             else:
-                documents.append(Document(**doc_data))
+                documents.append(Document(**normalize_document_input(doc_data)))
         
         # Create CrawlJob with current status and data
         return CrawlJob(

firecrawl/v2/methods/scrape.py

@@ -4,6 +4,7 @@ Scraping functionality for Firecrawl v2 API.
 
 from typing import Optional, Dict, Any
 from ..types import ScrapeOptions, Document
+from ..utils.normalize import normalize_document_input
 from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
 
 
@@ -59,10 +60,5 @@ def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None
         raise Exception(body.get("error", "Unknown error occurred"))
 
     document_data = body.get("data", {})
-    # Normalize keys for Document (no Pydantic aliases)
-    normalized = dict(document_data)
-    if 'rawHtml' in normalized and 'raw_html' not in normalized:
-        normalized['raw_html'] = normalized.pop('rawHtml')
-    if 'changeTracking' in normalized and 'change_tracking' not in normalized:
-        normalized['change_tracking'] = normalized.pop('changeTracking')
+    normalized = normalize_document_input(document_data)
     return Document(**normalized)

firecrawl/v2/methods/search.py

@@ -2,8 +2,10 @@
 Search functionality for Firecrawl v2 API.
 """
 
+import re
 from typing import Optional, Dict, Any, Union
 from ..types import SearchRequest, SearchData, SearchResult, Document
+from ..utils.normalize import normalize_document_input
 from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
 
 
@@ -50,12 +52,7 @@ def search(
                     if request.scrape_options is not None and any(
                         key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']
                     ):
-                        # Normalize keys for Document (no Pydantic aliases)
-                        normalized = dict(doc_data)
-                        if 'rawHtml' in normalized and 'raw_html' not in normalized:
-                            normalized['raw_html'] = normalized.pop('rawHtml')
-                        if 'changeTracking' in normalized and 'change_tracking' not in normalized:
-                            normalized['change_tracking'] = normalized.pop('changeTracking')
+                        normalized = normalize_document_input(doc_data)
                         results.append(Document(**normalized))
                     else:
                         # Minimal search result shape
@@ -123,11 +120,18 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:
     # Validate tbs (time-based search, if provided)
     if request.tbs is not None:
         valid_tbs_values = {
-            "qdr:d", "qdr:w", "qdr:m", "qdr:y",  # Google time filters
+            "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y",  # Google time filters
             "d", "w", "m", "y"  # Short forms
         }
-        if request.tbs not in valid_tbs_values:
-            raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values}")
+        
+        if request.tbs in valid_tbs_values:
+            pass  # Valid predefined value
+        elif request.tbs.startswith("cdr:"):
+            custom_date_pattern = r"^cdr:1,cd_min:\d{1,2}/\d{1,2}/\d{4},cd_max:\d{1,2}/\d{1,2}/\d{4}$"
+            if not re.match(custom_date_pattern, request.tbs):
+                raise ValueError(f"Invalid custom date range format: {request.tbs}. Expected format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
+        else:
+            raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values} or custom date range format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
     
     # Validate scrape_options (if provided)
     if request.scrape_options is not None:
@@ -170,4 +174,4 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
             data["scrapeOptions"] = scrape_data
         data.pop("scrape_options", None)
     
-    return data
+    return data

firecrawl/v2/types.py

@@ -7,7 +7,8 @@ This module contains clean, modern type definitions for the v2 API.
 import warnings
 from datetime import datetime
 from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union
-from pydantic import BaseModel, Field, field_validator
+import logging
+from pydantic import BaseModel, Field, field_validator, ValidationError
 
 # Suppress pydantic warnings about schema field shadowing
 # Tested using schema_field alias="schema" but it doesn't work.
@@ -19,6 +20,9 @@ warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" s
 
 T = TypeVar('T')
 
+# Module logger
+logger = logging.getLogger("firecrawl")
+
 # Base response types
 class BaseResponse(BaseModel, Generic[T]):
     """Base response structure for all API responses."""
@@ -29,18 +33,57 @@ class BaseResponse(BaseModel, Generic[T]):
 
 # Document and content types
 class DocumentMetadata(BaseModel):
-    """Metadata for scraped documents."""
+    """Metadata for scraped documents (snake_case only; API camelCase normalized in code)."""
+    # Common metadata fields
     title: Optional[str] = None
     description: Optional[str] = None
+    url: Optional[str] = None
     language: Optional[str] = None
     keywords: Optional[Union[str, List[str]]] = None
     robots: Optional[str] = None
+
+    # OpenGraph and social metadata
     og_title: Optional[str] = None
     og_description: Optional[str] = None
     og_url: Optional[str] = None
     og_image: Optional[str] = None
+    og_audio: Optional[str] = None
+    og_determiner: Optional[str] = None
+    og_locale: Optional[str] = None
+    og_locale_alternate: Optional[List[str]] = None
+    og_site_name: Optional[str] = None
+    og_video: Optional[str] = None
+
+    # Dublin Core and other site metadata
+    favicon: Optional[str] = None
+    dc_terms_created: Optional[str] = None
+    dc_date_created: Optional[str] = None
+    dc_date: Optional[str] = None
+    dc_terms_type: Optional[str] = None
+    dc_type: Optional[str] = None
+    dc_terms_audience: Optional[str] = None
+    dc_terms_subject: Optional[str] = None
+    dc_subject: Optional[str] = None
+    dc_description: Optional[str] = None
+    dc_terms_keywords: Optional[str] = None
+
+    modified_time: Optional[str] = None
+    published_time: Optional[str] = None
+    article_tag: Optional[str] = None
+    article_section: Optional[str] = None
+
+    # Response-level metadata
     source_url: Optional[str] = None
     status_code: Optional[int] = None
+    scrape_id: Optional[str] = None
+    num_pages: Optional[int] = None
+    content_type: Optional[str] = None
+    proxy_used: Optional[Literal["basic", "stealth"]] = None
+    cache_state: Optional[Literal["hit", "miss"]] = None
+    cached_at: Optional[str] = None
+    credits_used: Optional[int] = None
+
+    # Error information
     error: Optional[str] = None
 
     @staticmethod
@@ -85,6 +128,29 @@ class Document(BaseModel):
     warning: Optional[str] = None
     change_tracking: Optional[Dict[str, Any]] = None
 
+    @property
+    def metadata_typed(self) -> DocumentMetadata:
+        """Always returns a DocumentMetadata instance for LSP-friendly access."""
+        md = self.metadata
+        if isinstance(md, DocumentMetadata):
+            return md
+        if isinstance(md, dict):
+            try:
+                return DocumentMetadata(**md)
+            except (ValidationError, TypeError) as exc:
+                logger.debug("Failed to construct DocumentMetadata from dict: %s", exc)
+        return DocumentMetadata()
+
+    @property
+    def metadata_dict(self) -> Dict[str, Any]:
+        """Returns metadata as a plain dict (exclude None)."""
+        md = self.metadata
+        if isinstance(md, DocumentMetadata):
+            return md.model_dump(exclude_none=True)
+        if isinstance(md, dict):
+            return {k: v for k, v in md.items() if v is not None}
+        return {}
+
 # Webhook types
 class WebhookConfig(BaseModel):
     """Configuration for webhooks."""

firecrawl/v2/utils/normalize.py

@@ -0,0 +1,107 @@
+"""
+Normalization helpers for v2 API payloads to avoid relying on Pydantic aliases.
+"""
+
+from typing import Any, Dict, List
+from ..types import DocumentMetadata
+
+
+def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert API v2 camelCase metadata keys to snake_case expected by DocumentMetadata.
+    Leaves unknown keys as-is.
+    """
+    mapping = {
+        # OpenGraph
+        "ogTitle": "og_title",
+        "ogDescription": "og_description",
+        "ogUrl": "og_url",
+        "ogImage": "og_image",
+        "ogAudio": "og_audio",
+        "ogDeterminer": "og_determiner",
+        "ogLocale": "og_locale",
+        "ogLocaleAlternate": "og_locale_alternate",
+        "ogSiteName": "og_site_name",
+        "ogVideo": "og_video",
+        # Dublin Core and misc
+        "dcTermsCreated": "dc_terms_created",
+        "dcDateCreated": "dc_date_created",
+        "dcDate": "dc_date",
+        "dcTermsType": "dc_terms_type",
+        "dcType": "dc_type",
+        "dcTermsAudience": "dc_terms_audience",
+        "dcTermsSubject": "dc_terms_subject",
+        "dcSubject": "dc_subject",
+        "dcDescription": "dc_description",
+        "dcTermsKeywords": "dc_terms_keywords",
+        "modifiedTime": "modified_time",
+        "publishedTime": "published_time",
+        "articleTag": "article_tag",
+        "articleSection": "article_section",
+        # Response-level
+        "sourceURL": "source_url",
+        "statusCode": "status_code",
+        "scrapeId": "scrape_id",
+        "numPages": "num_pages",
+        "contentType": "content_type",
+        "proxyUsed": "proxy_used",
+        "cacheState": "cache_state",
+        "cachedAt": "cached_at",
+        "creditsUsed": "credits_used",
+    }
+
+    out: Dict[str, Any] = {}
+    for k, v in md.items():
+        snake = mapping.get(k, k)
+        out[snake] = v
+
+    # Light coercions where server may send strings/lists
+    if isinstance(out.get("status_code"), str):
+        try:
+            out["status_code"] = int(out["status_code"])  # type: ignore
+        except ValueError:
+            pass
+
+    # Generic rule: if a value is a list, join with ", " for string-like fields,
+    # except for explicit fields we preserve as lists.
+    preserve_list_fields: List[str] = [
+        "og_locale_alternate",
+    ]
+    for f, val in list(out.items()):
+        if isinstance(val, list) and f not in preserve_list_fields:
+            try:
+                out[f] = ", ".join(str(x) for x in val)
+            except Exception:
+                # Fallback: keep original list if join fails
+                pass
+
+    return out
+
+
+def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Normalize a raw Document dict from the API into the Python SDK's expected shape:
+    - Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking
+    - Convert metadata keys from camelCase to snake_case
+    """
+    normalized = dict(doc)
+
+    if "rawHtml" in normalized and "raw_html" not in normalized:
+        normalized["raw_html"] = normalized.pop("rawHtml")
+
+    if "changeTracking" in normalized and "change_tracking" not in normalized:
+        normalized["change_tracking"] = normalized.pop("changeTracking")
+
+    md = normalized.get("metadata")
+    if isinstance(md, dict):
+        mapped = _map_metadata_keys(md)
+        # Construct a concrete DocumentMetadata so downstream has a typed object
+        try:
+            normalized["metadata"] = DocumentMetadata(**mapped)
+        except Exception:
+            # Fallback to mapped dict if model construction fails for any reason
+            normalized["metadata"] = mapped
+
+    return normalized
+
+

firecrawl/v2/watcher.py

@@ -15,6 +15,7 @@ from typing import Callable, List, Optional, Literal, Union, Dict, Any
 import websockets
 
 from .types import CrawlJob, BatchScrapeJob, Document
+from .utils.normalize import normalize_document_input
 
 
 JobKind = Literal["crawl", "batch"]
@@ -172,11 +173,7 @@ class Watcher:
                         docs: List[Document] = []
                         for doc in self.data:
                             if isinstance(doc, dict):
-                                d = dict(doc)
-                                if "rawHtml" in d and "raw_html" not in d:
-                                    d["raw_html"] = d.pop("rawHtml")
-                                if "changeTracking" in d and "change_tracking" not in d:
-                                    d["change_tracking"] = d.pop("changeTracking")
+                                d = normalize_document_input(doc)
                                 docs.append(Document(**d))
                         if self._kind == "crawl":
                             job = CrawlJob(
@@ -212,11 +209,7 @@ class Watcher:
                         docs = []
                         for doc in payload.get("data", []):
                             if isinstance(doc, dict):
-                                d = dict(doc)
-                                if "rawHtml" in d and "raw_html" not in d:
-                                    d["raw_html"] = d.pop("rawHtml")
-                                if "changeTracking" in d and "change_tracking" not in d:
-                                    d["change_tracking"] = d.pop("changeTracking")
+                                d = normalize_document_input(doc)
                                 docs.append(Document(**d))
                         job = CrawlJob(
                             status=status_str,
@@ -241,11 +234,7 @@ class Watcher:
                         docs = []
                         for doc in payload.get("data", []):
                             if isinstance(doc, dict):
-                                d = dict(doc)
-                                if "rawHtml" in d and "raw_html" not in d:
-                                    d["raw_html"] = d.pop("rawHtml")
-                                if "changeTracking" in d and "change_tracking" not in d:
-                                    d["change_tracking"] = d.pop("changeTracking")
+                                d = normalize_document_input(doc)
                                 docs.append(Document(**d))
                         job = BatchScrapeJob(
                             status=status_str,

firecrawl/v2/watcher_async.py

@@ -16,6 +16,7 @@ import websockets
 from websockets.exceptions import ConnectionClosed, ConnectionClosedOK, ConnectionClosedError
 
 from .types import BatchScrapeJob, CrawlJob, Document
+from .utils.normalize import normalize_document_input
 
 JobKind = Literal["crawl", "batch"]
 
@@ -216,11 +217,7 @@ class AsyncWatcher:
         source_docs = docs_override if docs_override is not None else payload.get("data", []) or []
         for doc in source_docs:
             if isinstance(doc, dict):
-                d = dict(doc)
-                if "rawHtml" in d and "raw_html" not in d:
-                    d["raw_html"] = d.pop("rawHtml")
-                if "changeTracking" in d and "change_tracking" not in d:
-                    d["change_tracking"] = d.pop("changeTracking")
+                d = normalize_document_input(doc)
                 docs.append(Document(**d))
 
         if self._kind == "crawl":

{firecrawl_py-3.0.3.dist-info → firecrawl_py-3.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: firecrawl-py
-Version: 3.0.3
+Version: 3.1.1
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/firecrawl/firecrawl
 Author: Mendable.ai
@@ -40,6 +40,10 @@ Requires-Dist: websockets
 Requires-Dist: nest-asyncio
 Requires-Dist: pydantic
 Requires-Dist: aiohttp
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
 
 # Firecrawl Python SDK
 

{firecrawl_py-3.0.3.dist-info → firecrawl_py-3.1.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
-firecrawl/__init__.py,sha256=5w9g4kvMhD7vpl37kKrLpgrUEQ1OWfXyj4tzsvAiQyE,2191
-firecrawl/client.py,sha256=VxlMrvoq288KiIMKFk7Fq22KG0DGLZQQm56vilT71pQ,11058
+firecrawl/__init__.py,sha256=-BUn8vFl0b_PPw4VJ0lV9y0BKsHBlOSfxGTPqGMZK7U,2192
+firecrawl/client.py,sha256=2BGIRTiW2eR6q3wu_g2s3VTQtrHYauoDeNF1YklQpHo,11089
 firecrawl/firecrawl.backup.py,sha256=v1FEN3jR4g5Aupg4xp6SLkuFvYMQuUKND2YELbYjE6c,200430
 firecrawl/types.py,sha256=yZ4iza0M1T2kxNbt-tLEOKH7o6mFKZZ11VAZGodHSq4,2734
 firecrawl/__tests__/e2e/v2/conftest.py,sha256=I28TUpN5j0-9gM79NlbrDS8Jlsheao657od2f-2xK0Y,2587
@@ -13,7 +13,7 @@ firecrawl/__tests__/e2e/v2/test_search.py,sha256=MN-q82gHlm5DT2HsnAQgW1NwVbgowlF
 firecrawl/__tests__/e2e/v2/test_usage.py,sha256=JlBkYblhThua5qF2crRjsPpq4Ja0cBsdzxZ5zxXnQ_Y,805
 firecrawl/__tests__/e2e/v2/test_watcher.py,sha256=OPTKLhVAKWqXl2Tieo6zCN1xpEwZDsz-B977CVJgLMA,1932
 firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py,sha256=gJv_mLzzoAYftETB2TLkrpSfB5c04kaYgkD4hQTYsIg,2639
-firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py,sha256=AefCZA1he1UkGv80UXtyQru-zQbESd1L4yIAdycW-Y0,7317
+firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py,sha256=X-nk5tkYUYIkM6kTYl7GDjvxh2JT9GxJqk2KlO8xpWw,7282
 firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py,sha256=3CNRIFzgBMcOYOLhnKcK1k5a3Gy--u08EGDkL31uieM,1199
 firecrawl/__tests__/e2e/v2/aio/test_aio_map.py,sha256=nckl1kbiEaaTdu5lm__tOoTDG-txTYwwSH3KZEvyKzc,1199
 firecrawl/__tests__/e2e/v2/aio/test_aio_scrape.py,sha256=b17A7advBEjxrjdait2w8GHztZeKy_P3zZ3ixm5H7xw,4453
@@ -27,7 +27,7 @@ firecrawl/__tests__/unit/v2/methods/test_crawl_validation.py,sha256=kErOmHSD01eM
 firecrawl/__tests__/unit/v2/methods/test_map_request_preparation.py,sha256=toVcgnMp_cFeYsIUuyKGEWZGp0nAAkzaeFGUbY0zY0o,1868
 firecrawl/__tests__/unit/v2/methods/test_scrape_request_preparation.py,sha256=wDOslsA5BN4kyezlaT5GeMv_Ifn8f461EaA7i5ujnaQ,3482
 firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py,sha256=14lUgFpQsiosgMKjDustBRVE0zXnHujBI76F8BC5PZ4,6072
-firecrawl/__tests__/unit/v2/methods/test_search_validation.py,sha256=PaV_kSgzjW8A3eFBCCn1-y4WFZBR2nf84NZk4UEBPX8,8275
+firecrawl/__tests__/unit/v2/methods/test_search_validation.py,sha256=7UGcNHpQzCpZbAPYjthfdPFWmAPcoApY-ED-khtuANs,9498
 firecrawl/__tests__/unit/v2/methods/test_usage_types.py,sha256=cCHHfa6agSjD0brQ9rcAcw2kaI9riUH5C0dXV-fqktg,591
 firecrawl/__tests__/unit/v2/methods/test_webhook.py,sha256=AvvW-bKpUA--Lvtif2bmUIp-AxiaMJ29ie1i9dk8WbI,4586
 firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_params.py,sha256=9azJxVvDOBqUevLp-wBF9gF7Ptj-7nN6LOkPQncFX2M,456
@@ -45,34 +45,35 @@ firecrawl/v1/client.py,sha256=sydurfEFTsXyowyaGryA1lkPxN_r9Nf6iQpM43OwJyM,201672
 firecrawl/v2/__init__.py,sha256=Jc6a8tBjYG5OPkjDM5pl-notyys-7DEj7PLEfepv3fc,137
 firecrawl/v2/client.py,sha256=P6WAzwYGLLIANTrqAM-K4EUdGWQoFsi-zCjBibbxKQw,30507
 firecrawl/v2/client_async.py,sha256=zwxHis1bSh0tSF1480ze-4XDQEDJ5yDur1ZqtL94dwc,10127
-firecrawl/v2/types.py,sha256=Qj4kZ05rKKUQRzzcn4xZbcUZHgt6LXdG29T0NyZTPns,17866
-firecrawl/v2/watcher.py,sha256=tUPyYEGfQq93sAPDzxEbOmDTaUqW6pltKGHz2QdSGks,15063
-firecrawl/v2/watcher_async.py,sha256=yw3Jp_tNvTgR697AyNPYhAIb0vL5KixUwv2oVkHaQEA,10456
-firecrawl/v2/methods/batch.py,sha256=bTn9uMslVOfQdU3645kmt151t1j0suPPyNRgYM9zXHU,12165
-firecrawl/v2/methods/crawl.py,sha256=xrUe2y_T7ZYmAVll45Gag7BdS-Mmd1XxbJ88hXAMSuI,15404
+firecrawl/v2/types.py,sha256=zV0XAX_pJaJj41uxfJewKPANxd45BCL48nvbN_ybLOc,20222
+firecrawl/v2/watcher.py,sha256=FOU71tqSKxgeuGycu4ye0SLc2dw7clIcoQjPsi-4Csc,14229
+firecrawl/v2/watcher_async.py,sha256=AVjW2mgABniolSsauK4u0FW8ya6WzRUdyEg2R-8vGCw,10278
+firecrawl/v2/methods/batch.py,sha256=us7zUGl7u9ZDIEk2J3rNqj87bkaNjXU27SMFW_fdcg8,11932
+firecrawl/v2/methods/crawl.py,sha256=4ZUmanHNuNtq9wbKMAZ3lenuPcNdOaV0kYXqMI5XJJ8,15485
 firecrawl/v2/methods/extract.py,sha256=-Jr4BtraU3b7hd3JIY73V-S69rUclxyXyUpoQb6DCQk,4274
 firecrawl/v2/methods/map.py,sha256=4SADb0-lkbdOWDmO6k8_TzK0yRti5xsN40N45nUl9uA,2592
-firecrawl/v2/methods/scrape.py,sha256=Sd3KNNCmSXvR17yLB72OEPeVPKk5DGM-chkm0WpYtSk,2178
-firecrawl/v2/methods/search.py,sha256=Ou0R_3rO0co7BJx3XBhiTX4bXPFlFIuU8b68bzaFMes,6488
+firecrawl/v2/methods/scrape.py,sha256=CSHBwC-P91UfrW3zHirjNAs2h899FKcWvd1DY_4fJdo,1921
+firecrawl/v2/methods/search.py,sha256=HB17OorEHfZXZh8tvfSqVKxS9uYtqBX3Me4YAFMF7w0,6640
 firecrawl/v2/methods/usage.py,sha256=OJlkxwaB-AAtgO3WLr9QiqBRmjdh6GVhroCgleegupQ,1460
 firecrawl/v2/methods/aio/__init__.py,sha256=RocMJnGwnLIvGu3G8ZvY8INkipC7WHZiu2bE31eSyJs,35
 firecrawl/v2/methods/aio/batch.py,sha256=GS_xsd_Uib1fxFITBK1sH88VGzFMrIcqJVQqOvMQ540,3735
-firecrawl/v2/methods/aio/crawl.py,sha256=PEFIqZ7UtTBZSbs3fQuoxWMN68WbsTcj2AnZZvnfggk,6936
+firecrawl/v2/methods/aio/crawl.py,sha256=pC6bHVk30Hj1EJdAChxpMOg0Xx_GVqq4tIlvU2e5RQ4,6688
 firecrawl/v2/methods/aio/extract.py,sha256=IfNr2ETqt4dR73JFzrEYI4kk5vpKnJOG0BmPEjGEoO4,4217
 firecrawl/v2/methods/aio/map.py,sha256=EuT-5A0cQr_e5SBfEZ6pnl8u0JUwEEvSwhyT2N-QoKU,2326
-firecrawl/v2/methods/aio/scrape.py,sha256=-VLFlE7Ma9TS0yXN4esvC3CPCic1Kq6d5Hc4j8CudGc,1586
-firecrawl/v2/methods/aio/search.py,sha256=UA5KJlzOvSqc3TYEm1wnZLNVoP4SsrRwdQbXGe7UcDk,2781
+firecrawl/v2/methods/aio/scrape.py,sha256=ilA9qco8YGwCFpE0PN1XBQUyuHPQwH2QioZ-xsfxhgU,1386
+firecrawl/v2/methods/aio/search.py,sha256=nuRmFCA_ymBw2tXJZ88vjZY-BueIRNonkSsrxExwusM,2501
 firecrawl/v2/methods/aio/usage.py,sha256=OtBi6X-aT09MMR2dpm3vBCm9JrJZIJLCQ8jJ3L7vie4,1606
 firecrawl/v2/utils/__init__.py,sha256=i1GgxySmqEXpWSBQCu3iZBPIJG7fXj0QXCDWGwerWNs,338
 firecrawl/v2/utils/error_handler.py,sha256=Iuf916dHphDY8ObNNlWy75628DFeJ0Rv8ljRp4LttLE,4199
 firecrawl/v2/utils/get_version.py,sha256=0CxW_41q2hlzIxEWOivUCaYw3GFiSIH32RPUMcIgwAY,492
 firecrawl/v2/utils/http_client.py,sha256=_n8mp4xi6GGihg662Lsv6TSlvw9zykyADwEk0fg8mYA,4873
 firecrawl/v2/utils/http_client_async.py,sha256=P4XG6nTz6kKH3vCPTz6i7DRhbpK4IImRGaFvQFGBFRc,1874
+firecrawl/v2/utils/normalize.py,sha256=nlTU6QRghT1YKZzNZlIQj4STSRuSUGrS9cCErZIcY5w,3636
 firecrawl/v2/utils/validation.py,sha256=L8by7z-t6GuMGIYkK7il1BM8d-4_-sAdG9hDMF_LeG4,14518
+firecrawl_py-3.1.1.dist-info/licenses/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
 tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
 tests/test_timeout_conversion.py,sha256=PWlIEMASQNhu4cp1OW_ebklnE9NCiigPnEFCtI5N3w0,3996
-firecrawl_py-3.0.3.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
-firecrawl_py-3.0.3.dist-info/METADATA,sha256=zHXDvMswdqaodPcxfqZRBkrALt0WU2mN6tEukpD_UiM,7308
-firecrawl_py-3.0.3.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-firecrawl_py-3.0.3.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
-firecrawl_py-3.0.3.dist-info/RECORD,,
+firecrawl_py-3.1.1.dist-info/METADATA,sha256=JGp1wtMdsywSEAY520vzMJtol2Mq14v3CbC_ec6DSKA,7390
+firecrawl_py-3.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+firecrawl_py-3.1.1.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl_py-3.1.1.dist-info/RECORD,,

{firecrawl_py-3.0.3.dist-info → firecrawl_py-3.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any