firecrawl-py 2.16.3__py3-none-any.whl → 3.0.2__py3-none-any.whl

build/lib/firecrawl/firecrawl.py → firecrawl/firecrawl.backup.py

@@ -24,12 +24,6 @@ import aiohttp
 import asyncio
 from pydantic import Field
 
-# Suppress Pydantic warnings about attribute shadowing
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"FirecrawlDocument\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"ChangeTrackingData\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonConfig\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ExtractParams\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingOptions\" shadows an attribute in parent \"BaseModel\"")
 
 def get_version():
   try:
@@ -106,7 +100,7 @@ class ChangeTrackingData(pydantic.BaseModel):
     changeStatus: str  # "new" | "same" | "changed" | "removed"
     visibility: str  # "visible" | "hidden"
     diff: Optional[Dict[str, Any]] = None
-    json: Optional[Any] = None
+    json_field: Optional[Any] = pydantic.Field(None, alias='json')
 
 class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
     """Document retrieved or processed by Firecrawl."""
@@ -116,7 +110,7 @@ class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
     rawHtml: Optional[str] = None
     links: Optional[List[str]] = None
     extract: Optional[T] = None
-    json: Optional[T] = None
+    json_field: Optional[T] = pydantic.Field(None, alias='json')
     screenshot: Optional[str] = None
     metadata: Optional[Any] = None
     actions: Optional[ActionsResult] = None
@@ -139,7 +133,7 @@ class WebhookConfig(pydantic.BaseModel):
 class ChangeTrackingOptions(pydantic.BaseModel):
     """Configuration for change tracking."""
     modes: Optional[List[Literal["git-diff", "json"]]] = None
-    schema: Optional[Any] = None
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     prompt: Optional[str] = None
     tag: Optional[str] = None
 
@@ -151,7 +145,7 @@ class ScrapeOptions(pydantic.BaseModel):
     excludeTags: Optional[List[str]] = None
     onlyMainContent: Optional[bool] = None
     waitFor: Optional[int] = None
-    timeout: Optional[int] = None
+    timeout: Optional[int] = 30000
     location: Optional[LocationConfig] = None
     mobile: Optional[bool] = None
     skipTlsVerification: Optional[bool] = None
@@ -219,7 +213,7 @@ class ExtractAgent(pydantic.BaseModel):
 class JsonConfig(pydantic.BaseModel):
     """Configuration for extraction."""
     prompt: Optional[str] = None
-    schema: Optional[Any] = None
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     systemPrompt: Optional[str] = None
     agent: Optional[ExtractAgent] = None
 
@@ -264,6 +258,7 @@ class CrawlParams(pydantic.BaseModel):
     maxDiscoveryDepth: Optional[int] = None
     limit: Optional[int] = None
     allowBackwardLinks: Optional[bool] = None
+    crawlEntireDomain: Optional[bool] = None
     allowExternalLinks: Optional[bool] = None
     ignoreSitemap: Optional[bool] = None
     scrapeOptions: Optional[ScrapeOptions] = None
@@ -305,7 +300,7 @@ class MapParams(pydantic.BaseModel):
     includeSubdomains: Optional[bool] = None
     sitemapOnly: Optional[bool] = None
     limit: Optional[int] = None
-    timeout: Optional[int] = None
+    timeout: Optional[int] = 30000
     useIndex: Optional[bool] = None
 
 class MapResponse(pydantic.BaseModel):
@@ -317,7 +312,7 @@ class MapResponse(pydantic.BaseModel):
 class ExtractParams(pydantic.BaseModel):
     """Parameters for extracting information from URLs."""
     prompt: Optional[str] = None
-    schema: Optional[Any] = None
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     systemPrompt: Optional[str] = None
     allowExternalLinks: Optional[bool] = None
     enableWebSearch: Optional[bool] = None
@@ -431,7 +426,7 @@ class ExtractParams(pydantic.BaseModel):
     Parameters for the extract operation.
     """
     prompt: Optional[str] = None
-    schema: Optional[Any] = pydantic.Field(None, alias='schema')
+    schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
     system_prompt: Optional[str] = None
     allow_external_links: Optional[bool] = False
     enable_web_search: Optional[bool] = False
@@ -469,7 +464,7 @@ class FirecrawlApp:
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
             wait_for: Optional[int] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             location: Optional[LocationConfig] = None,
             mobile: Optional[bool] = None,
             skip_tls_verification: Optional[bool] = None,
@@ -484,6 +479,7 @@ class FirecrawlApp:
             max_age: Optional[int] = None,
             store_in_cache: Optional[bool] = None,
             zero_data_retention: Optional[bool] = None,
+            agent: Optional[AgentOptions] = None,
             **kwargs) -> ScrapeResponse[Any]:
         """
         Scrape and extract content from a URL.
@@ -508,6 +504,7 @@ class FirecrawlApp:
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
           change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings
           zero_data_retention (Optional[bool]): Whether to delete data after scrape is done
+          agent (Optional[AgentOptions]): Agent configuration for FIRE-1 model
 
 
         Returns:
@@ -547,7 +544,7 @@ class FirecrawlApp:
         if timeout:
             scrape_params['timeout'] = timeout
         if location:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -564,22 +561,24 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if change_tracking_options:
-            scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(exclude_none=True)
+            scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(by_alias=True, exclude_none=True)
         if max_age is not None:
             scrape_params['maxAge'] = max_age
         if store_in_cache is not None:
             scrape_params['storeInCache'] = store_in_cache
         if zero_data_retention is not None:
             scrape_params['zeroDataRetention'] = zero_data_retention
+        if agent is not None:
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         
         scrape_params.update(kwargs)
 
@@ -593,7 +592,7 @@ class FirecrawlApp:
             f'{self.api_url}/v1/scrape',
             headers=_headers,
             json=scrape_params,
-            timeout=(timeout + 5000 if timeout else None)
+            timeout=(timeout / 1000.0 + 5 if timeout is not None else None)
         )
 
         if response.status_code == 200:
@@ -620,7 +619,7 @@ class FirecrawlApp:
             lang: Optional[str] = None,
             country: Optional[str] = None,
             location: Optional[str] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             scrape_options: Optional[ScrapeOptions] = None,
             **kwargs) -> SearchResponse:
         """
@@ -670,7 +669,7 @@ class FirecrawlApp:
         if timeout is not None:
             search_params['timeout'] = timeout
         if scrape_options is not None:
-            search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         
         # Add any additional kwargs
         search_params.update(kwargs)
@@ -678,7 +677,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = SearchParams(query=query, **search_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['origin'] = f"python-sdk@{version}"
 
         if _integration:
@@ -792,7 +791,7 @@ class FirecrawlApp:
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -815,7 +814,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -921,7 +920,7 @@ class FirecrawlApp:
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -943,7 +942,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1161,6 +1160,7 @@ class FirecrawlApp:
             max_discovery_depth=max_discovery_depth,
             limit=limit,
             allow_backward_links=allow_backward_links,
+            crawl_entire_domain=crawl_entire_domain,
             allow_external_links=allow_external_links,
             ignore_sitemap=ignore_sitemap,
             scrape_options=scrape_options,
@@ -1189,7 +1189,7 @@ class FirecrawlApp:
             include_subdomains: Optional[bool] = None,
             sitemap_only: Optional[bool] = None,
             limit: Optional[int] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             use_index: Optional[bool] = None,
             **kwargs) -> MapResponse:
         """
@@ -1242,7 +1242,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = MapParams(**map_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1280,7 +1280,7 @@ class FirecrawlApp:
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -1354,7 +1354,7 @@ class FirecrawlApp:
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -1369,16 +1369,16 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if max_concurrency is not None:
             scrape_params['maxConcurrency'] = max_concurrency
         if zero_data_retention is not None:
@@ -1389,7 +1389,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1421,7 +1421,7 @@ class FirecrawlApp:
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -1495,7 +1495,7 @@ class FirecrawlApp:
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -1510,16 +1510,16 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if max_concurrency is not None:
             scrape_params['maxConcurrency'] = max_concurrency
         if zero_data_retention is not None:
@@ -1530,7 +1530,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -1561,7 +1561,7 @@ class FirecrawlApp:
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -1631,7 +1631,7 @@ class FirecrawlApp:
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -1646,16 +1646,16 @@ class FirecrawlApp:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if max_concurrency is not None:
             scrape_params['maxConcurrency'] = max_concurrency
         if zero_data_retention is not None:
@@ -1666,7 +1666,7 @@ class FirecrawlApp:
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -2095,7 +2095,7 @@ class FirecrawlApp:
         )
 
         headers = self._prepare_headers()
-        json_data = {'url': url, **params.dict(exclude_none=True)}
+        json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
@@ -2204,7 +2204,7 @@ class FirecrawlApp:
             requests.RequestException: If the request fails after the specified retries.
         """
         for attempt in range(retries):
-            response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] + 5000) if "timeout" in data else None))
+            response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] / 1000.0 + 5) if "timeout" in data and data["timeout"] is not None else None))
             if response.status_code == 502:
                 time.sleep(backoff_factor * (2 ** attempt))
             else:
@@ -2336,10 +2336,22 @@ class FirecrawlApp:
             Exception: An exception with a message containing the status code and error details from the response.
         """
         try:
-            error_message = response.json().get('error', 'No error message provided.')
-            error_details = response.json().get('details', 'No additional error details provided.')
+            response_json = response.json()
+            error_message = response_json.get('error', 'No error message provided.')
+            error_details = response_json.get('details', 'No additional error details provided.')
         except:
-            raise requests.exceptions.HTTPError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status_code}', response=response)
+            # If we can't parse JSON, provide a helpful error message with response content
+            try:
+                response_text = response.text[:500]  # Limit to first 500 chars
+                if response_text.strip():
+                    error_message = f"Server returned non-JSON response: {response_text}"
+                    error_details = f"Full response status: {response.status_code}"
+                else:
+                    error_message = f"Server returned empty response with status {response.status_code}"
+                    error_details = "No additional details available"
+            except ValueError:
+                error_message = f"Server returned unreadable response with status {response.status_code}"
+                error_details = "No additional details available"
         
         message = self._get_error_message(response.status_code, action, error_message, error_details)
 
@@ -2362,7 +2374,7 @@ class FirecrawlApp:
         if status_code == 402:
             return f"Payment Required: Failed to {action}. {error_message} - {error_details}"
         elif status_code == 403:
-            message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+            return f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
         elif status_code == 408:
             return f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
         elif status_code == 409:
@@ -2516,7 +2528,7 @@ class FirecrawlApp:
 
         headers = self._prepare_headers()
         
-        json_data = {'query': query, **research_params.dict(exclude_none=True)}
+        json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         # Handle json options schema if present
@@ -2600,7 +2612,7 @@ class FirecrawlApp:
         method_params = {
             "scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for", 
                           "timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",
-                          "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "integration"},
+                          "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "agent", "integration"},
             "search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options", "integration"},
             "crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",
                          "allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",
@@ -2975,7 +2987,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
             wait_for: Optional[int] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             location: Optional[LocationConfig] = None,
             mobile: Optional[bool] = None,
             skip_tls_verification: Optional[bool] = None,
@@ -2986,6 +2998,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract: Optional[JsonConfig] = None,
             json_options: Optional[JsonConfig] = None,
             actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,
+            agent: Optional[AgentOptions] = None,
             **kwargs) -> ScrapeResponse[Any]:
         """
         Scrape a single URL asynchronously.
@@ -3008,6 +3021,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
           extract (Optional[JsonConfig]): Content extraction settings
           json_options (Optional[JsonConfig]): JSON extraction settings
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
+          agent (Optional[AgentOptions]): Agent configuration for FIRE-1 model
           **kwargs: Additional parameters to pass to the API
 
         Returns:
@@ -3052,7 +3066,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout:
             scrape_params['timeout'] = timeout
         if location:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -3069,14 +3083,16 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+        if agent is not None:
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:
             scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])
         if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:
@@ -3109,7 +3125,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -3180,7 +3196,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -3195,22 +3211,23 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
-
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+        if actions is not None:
+            scrape_params['actions'] = [action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
 
         # Add any additional kwargs
         scrape_params.update(kwargs)
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -3247,7 +3264,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         exclude_tags: Optional[List[str]] = None,
         only_main_content: Optional[bool] = None,
         wait_for: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         location: Optional[LocationConfig] = None,
         mobile: Optional[bool] = None,
         skip_tls_verification: Optional[bool] = None,
@@ -3319,7 +3336,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout is not None:
             scrape_params['timeout'] = timeout
         if location is not None:
-            scrape_params['location'] = location.dict(exclude_none=True)
+            scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
         if mobile is not None:
             scrape_params['mobile'] = mobile
         if skip_tls_verification is not None:
@@ -3334,16 +3351,16 @@ class AsyncFirecrawlApp(FirecrawlApp):
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
                 extract["schema"] = self._ensure_schema_dict(extract["schema"])
-            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(exclude_none=True)
+            scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
         if json_options is not None:
             json_options = self._ensure_schema_dict(json_options)
             if isinstance(json_options, dict) and "schema" in json_options:
                 json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
-            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(exclude_none=True)
+            scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
         if actions:
-            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(exclude_none=True) for action in actions]
+            scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
         if agent is not None:
-            scrape_params['agent'] = agent.dict(exclude_none=True)
+            scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
         if zero_data_retention is not None:
             scrape_params['zeroDataRetention'] = zero_data_retention
         
@@ -3352,7 +3369,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = ScrapeParams(**scrape_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['urls'] = urls
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -3460,7 +3477,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -3479,7 +3496,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
         # Make request
@@ -3575,7 +3592,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if ignore_sitemap is not None:
             crawl_params['ignoreSitemap'] = ignore_sitemap
         if scrape_options is not None:
-            crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         if webhook is not None:
             crawl_params['webhook'] = webhook
         if deduplicate_similar_urls is not None:
@@ -3594,7 +3611,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = CrawlParams(**crawl_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -3732,7 +3749,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         include_subdomains: Optional[bool] = None,
         sitemap_only: Optional[bool] = None,
         limit: Optional[int] = None,
-        timeout: Optional[int] = None,
+        timeout: Optional[int] = 30000,
         params: Optional[MapParams] = None) -> MapResponse:
         """
         Asynchronously map and discover links from a URL.
@@ -3760,7 +3777,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         """
         map_params = {}
         if params:
-            map_params.update(params.dict(exclude_none=True))
+            map_params.update(params.dict(by_alias=True, exclude_none=True))
 
         # Add individual parameters
         if search is not None:
@@ -3778,7 +3795,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         # Create final params object
         final_params = MapParams(**map_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
 
@@ -4162,7 +4179,6 @@ class AsyncFirecrawlApp(FirecrawlApp):
             url,
             max_urls=max_urls,
             show_full_text=show_full_text,
-            cache=cache,
             experimental_stream=experimental_stream
         )
         if not response.get('success') or 'id' not in response:
@@ -4226,7 +4242,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         )
 
         headers = self._prepare_headers()
-        json_data = {'url': url, **params.dict(exclude_none=True)}
+        json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
@@ -4411,7 +4427,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
         headers = self._prepare_headers()
         
-        json_data = {'query': query, **research_params.dict(exclude_none=True)}
+        json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
@@ -4467,7 +4483,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             lang: Optional[str] = None,
             country: Optional[str] = None,
             location: Optional[str] = None,
-            timeout: Optional[int] = None,
+            timeout: Optional[int] = 30000,
             scrape_options: Optional[ScrapeOptions] = None,
             params: Optional[Union[Dict[str, Any], SearchParams]] = None,
             **kwargs) -> SearchResponse:
@@ -4503,7 +4519,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             if isinstance(params, dict):
                 search_params.update(params)
             else:
-                search_params.update(params.dict(exclude_none=True))
+                search_params.update(params.dict(by_alias=True, exclude_none=True))
 
         # Add individual parameters
         if limit is not None:
@@ -4521,14 +4537,14 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if timeout is not None:
             search_params['timeout'] = timeout
         if scrape_options is not None:
-            search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)
+            search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
         
         # Add any additional kwargs
         search_params.update(kwargs)
 
         # Create final params object
         final_params = SearchParams(query=query, **search_params)
-        params_dict = final_params.dict(exclude_none=True)
+        params_dict = final_params.dict(by_alias=True, exclude_none=True)
         params_dict['origin'] = f"python-sdk@{version}"
 
         return await self._async_post_request(