firecrawl 2.16.2__py3-none-any.whl → 2.16.5__py3-none-any.whl

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
 
-__version__ = "2.16.2"
+__version__ = "2.16.5"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/firecrawl.py

@@ -24,12 +24,6 @@ import aiohttp
 import asyncio
 from pydantic import Field
 
-# Suppress Pydantic warnings about attribute shadowing
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"FirecrawlDocument\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"ChangeTrackingData\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonConfig\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ExtractParams\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingOptions\" shadows an attribute in parent \"BaseModel\"")
 
 def get_version():
   try:
@@ -106,7 +100,7 @@ class ChangeTrackingData(pydantic.BaseModel):
     changeStatus: str  # "new" | "same" | "changed" | "removed"
     visibility: str  # "visible" | "hidden"
     diff: Optional[Dict[str, Any]] = None
-    json: Optional[Any] = None
+    json_field: Optional[Any] = pydantic.Field(None, alias='json')
 
 class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
     """Document retrieved or processed by Firecrawl."""
@@ -116,7 +110,7 @@ class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
     rawHtml: Optional[str] = None
     links: Optional[List[str]] = None
     extract: Optional[T] = None
-    json: Optional[T] = None
+    json_field: Optional[T] = pydantic.Field(None, alias='json')
     screenshot: Optional[str] = None
     metadata: Optional[Any] = None
     actions: Optional[ActionsResult] = None
@@ -139,7 +133,7 @@ class WebhookConfig(pydantic.BaseModel):
 class ChangeTrackingOptions(pydantic.BaseModel):
     """Configuration for change tracking."""
     modes: Optional[List[Literal["git-diff", "json"]]] = None
-    schema: Optional[Any] = None
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     prompt: Optional[str] = None
     tag: Optional[str] = None
 
@@ -219,7 +213,7 @@ class ExtractAgent(pydantic.BaseModel):
 class JsonConfig(pydantic.BaseModel):
     """Configuration for extraction."""
     prompt: Optional[str] = None
-    schema: Optional[Any] = None
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     systemPrompt: Optional[str] = None
     agent: Optional[ExtractAgent] = None
 
@@ -317,7 +311,7 @@ class MapResponse(pydantic.BaseModel):
 class ExtractParams(pydantic.BaseModel):
     """Parameters for extracting information from URLs."""
     prompt: Optional[str] = None
-    schema: Optional[Any] = None
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     systemPrompt: Optional[str] = None
     allowExternalLinks: Optional[bool] = None
     enableWebSearch: Optional[bool] = None
@@ -431,7 +425,7 @@ class ExtractParams(pydantic.BaseModel):
     Parameters for the extract operation.
     """
     prompt: Optional[str] = None
-    schema: Optional[Any] = pydantic.Field(None, alias='schema')
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     system_prompt: Optional[str] = None
     allow_external_links: Optional[bool] = False
     enable_web_search: Optional[bool] = False
@@ -484,6 +478,7 @@ class FirecrawlApp:
             max_age: Optional[int] = None,
             store_in_cache: Optional[bool] = None,
             zero_data_retention: Optional[bool] = None,
+            agent: Optional[AgentOptions] = None,
             **kwargs) -> ScrapeResponse[Any]:
         """
         Scrape and extract content from a URL.
@@ -508,6 +503,7 @@ class FirecrawlApp:
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
           change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings
           zero_data_retention (Optional[bool]): Whether to delete data after scrape is done
+          agent (Optional[AgentOptions]): Agent configuration for FIRE-1 model
 
 
         Returns:
@@ -520,6 +516,9 @@ class FirecrawlApp:
         Raises:
           Exception: If scraping fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "scrape_url")
+        
         _headers = self._prepare_headers()
 
         # Build scrape parameters
@@ -544,7 +543,7 @@ class FirecrawlApp:
         if timeout:
             scrape_params['timeout'] = timeout
         if location:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -561,22 +560,24 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if change_tracking_options:
-            scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(exclude_none=True)
+            scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(by_alias=True, exclude_none=True)
         if max_age is not None:
             scrape_params['maxAge'] = max_age
         if store_in_cache is not None:
             scrape_params['storeInCache'] = store_in_cache
         if zero_data_retention is not None:
             scrape_params['zeroDataRetention'] = zero_data_retention
+        if agent is not None:
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         
         scrape_params.update(kwargs)
 
@@ -667,7 +668,7 @@ class FirecrawlApp:
         if timeout is not None:
             search_params['timeout'] = timeout
         if scrape_options is not None:
-            search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         
         # Add any additional kwargs
         search_params.update(kwargs)
@@ -675,7 +676,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = SearchParams(query=query, **search_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['origin'] = f"python-sdk@{version}"
 
         if _integration:
@@ -789,7 +790,7 @@ class FirecrawlApp:
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -812,7 +813,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -918,7 +919,7 @@ class FirecrawlApp:
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -940,7 +941,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1239,7 +1240,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = MapParams(**map_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1351,7 +1352,7 @@ class FirecrawlApp:
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -1366,16 +1367,16 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if max_concurrency is not None:
             scrape_params['maxConcurrency'] = max_concurrency
         if zero_data_retention is not None:
@@ -1386,7 +1387,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1492,7 +1493,7 @@ class FirecrawlApp:
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -1507,16 +1508,16 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if max_concurrency is not None:
             scrape_params['maxConcurrency'] = max_concurrency
         if zero_data_retention is not None:
@@ -1527,7 +1528,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1628,7 +1629,7 @@ class FirecrawlApp:
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -1643,16 +1644,16 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if max_concurrency is not None:
             scrape_params['maxConcurrency'] = max_concurrency
         if zero_data_retention is not None:
@@ -1663,7 +1664,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -2092,7 +2093,7 @@ class FirecrawlApp:
         )
 
         headers = self._prepare_headers()
-        json_data = {'url': url, **params.dict(exclude_none=True)}
+        json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
@@ -2333,10 +2334,22 @@ class FirecrawlApp:
             Exception: An exception with a message containing the status code and error details from the response.
         """
         try:
-            error_message = response.json().get('error', 'No error message provided.')
-            error_details = response.json().get('details', 'No additional error details provided.')
+            response_json = response.json()
+            error_message = response_json.get('error', 'No error message provided.')
+            error_details = response_json.get('details', 'No additional error details provided.')
         except:
-            raise requests.exceptions.HTTPError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status_code}', response=response)
+            # If we can't parse JSON, provide a helpful error message with response content
+            try:
+                response_text = response.text[:500]  # Limit to first 500 chars
+                if response_text.strip():
+                    error_message = f"Server returned non-JSON response: {response_text}"
+                    error_details = f"Full response status: {response.status_code}"
+                else:
+                    error_message = f"Server returned empty response with status {response.status_code}"
+                    error_details = "No additional details available"
+            except ValueError:
+                error_message = f"Server returned unreadable response with status {response.status_code}"
+                error_details = "No additional details available"
         
         message = self._get_error_message(response.status_code, action, error_message, error_details)
 
@@ -2359,7 +2372,7 @@ class FirecrawlApp:
         if status_code == 402:
             return f"Payment Required: Failed to {action}. {error_message} - {error_details}"
         elif status_code == 403:
-            message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+            return f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
         elif status_code == 408:
             return f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
         elif status_code == 409:
@@ -2513,7 +2526,7 @@ class FirecrawlApp:
 
         headers = self._prepare_headers()
         
-        json_data = {'query': query, **research_params.dict(exclude_none=True)}
+        json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         # Handle json options schema if present
@@ -2597,7 +2610,7 @@ class FirecrawlApp:
         method_params = {
             "scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for", 
                           "timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",
-                          "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "integration"},
+                          "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "agent", "integration"},
             "search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options", "integration"},
             "crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",
                          "allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",
@@ -2983,6 +2996,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract: Optional[JsonConfig] = None,
             json_options: Optional[JsonConfig] = None,
             actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,
+            agent: Optional[AgentOptions] = None,
             **kwargs) -> ScrapeResponse[Any]:
         """
         Scrape a single URL asynchronously.
@@ -3005,6 +3019,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
           extract (Optional[JsonConfig]): Content extraction settings
           json_options (Optional[JsonConfig]): JSON extraction settings
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
+          agent (Optional[AgentOptions]): Agent configuration for FIRE-1 model
           **kwargs: Additional parameters to pass to the API
 
         Returns:
@@ -3049,7 +3064,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout:
             scrape_params['timeout'] = timeout
         if location:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -3066,14 +3081,16 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+        if agent is not None:
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:
             scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])
         if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:
@@ -3177,7 +3194,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -3192,22 +3209,23 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
-
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+        if actions is not None:
+            scrape_params['actions'] = [action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
 
         # Add any additional kwargs
         scrape_params.update(kwargs)
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -3316,7 +3334,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -3331,16 +3349,16 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if zero_data_retention is not None:
             scrape_params['zeroDataRetention'] = zero_data_retention
         
@@ -3349,7 +3367,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -3457,7 +3475,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -3476,7 +3494,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
         # Make request
@@ -3572,7 +3590,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -3591,7 +3609,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -3757,7 +3775,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         """
         map_params = {}
         if params:
-            map_params.update(params.dict(exclude_none=True))
+            map_params.update(params.dict(by_alias=True, exclude_none=True))
 
         # Add individual parameters
         if search is not None:
@@ -3775,7 +3793,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = MapParams(**map_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -4159,7 +4177,6 @@ class AsyncFirecrawlApp(FirecrawlApp):
             url,
             max_urls=max_urls,
             show_full_text=show_full_text,
-            cache=cache,
             experimental_stream=experimental_stream
         )
         if not response.get('success') or 'id' not in response:
@@ -4223,7 +4240,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         )
 
         headers = self._prepare_headers()
-        json_data = {'url': url, **params.dict(exclude_none=True)}
+        json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
@@ -4408,7 +4425,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         headers = self._prepare_headers()
         
-        json_data = {'query': query, **research_params.dict(exclude_none=True)}
+        json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
@@ -4500,7 +4517,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             if isinstance(params, dict):
                 search_params.update(params)
             else:
-                search_params.update(params.dict(exclude_none=True))
+                search_params.update(params.dict(by_alias=True, exclude_none=True))
 
         # Add individual parameters
         if limit is not None:
@@ -4518,14 +4535,14 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout is not None:
             search_params['timeout'] = timeout
         if scrape_options is not None:
-            search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         
         # Add any additional kwargs
         search_params.update(kwargs)
 
         # Create final params object
         final_params = SearchParams(query=query, **search_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['origin'] = f"python-sdk@{version}"
 
         return await self._async_post_request(

{firecrawl-2.16.2.dist-info → firecrawl-2.16.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 2.16.2
+Version: 2.16.5
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

{firecrawl-2.16.2.dist-info → firecrawl-2.16.5.dist-info}/RECORD

@@ -1,12 +1,12 @@
-firecrawl/__init__.py,sha256=WMYkjsU2CCNa2TAjTxeB_Z_2wAFOkK2Ax4sJNwUHPVU,2613
-firecrawl/firecrawl.py,sha256=oXuCZklh-UufHdIzpGhkxV5E4GhXTWOM6E0ubUDWgAE,198527
+firecrawl/__init__.py,sha256=M6X3Rk7QgbwslQRTX4yMw4TRAVtaD-11pQwfkleI3oc,2613
+firecrawl/firecrawl.py,sha256=RXURxzCDbKJ17KVzXwbo6Iy72c9Q1OgjtwRCYaphNwU,200263
 firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
 firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=k9IsEbdTHL9Cu49M4FpnQDEo2rnG6RqwmZAsK_EVJr4,21069
 tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
-firecrawl-2.16.2.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
-firecrawl-2.16.2.dist-info/METADATA,sha256=YmN8NqbgHCZFjJRTUNvJV2ltAro2CIK3bIAuvMnHIZU,7166
-firecrawl-2.16.2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-firecrawl-2.16.2.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
-firecrawl-2.16.2.dist-info/RECORD,,
+firecrawl-2.16.5.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl-2.16.5.dist-info/METADATA,sha256=kaGelYV72SrE23PK8xFBPcyBRAOMylHCQePSlFZO5B0,7166
+firecrawl-2.16.5.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl-2.16.5.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl-2.16.5.dist-info/RECORD,,