firecrawl 2.0.1__tar.gz → 2.1.0__tar.gz

{firecrawl-2.0.1 → firecrawl-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 2.0.1
+Version: 2.1.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

{firecrawl-2.0.1 → firecrawl-2.1.0}/firecrawl/__init__.py

@@ -11,9 +11,9 @@ For more information visit https://github.com/firecrawl/
 import logging
 import os
 
-from .firecrawl import FirecrawlApp, ExtractConfig # noqa
+from .firecrawl import FirecrawlApp, JsonConfig, ScrapeOptions # noqa
 
-__version__ = "2.0.1"
+__version__ = "2.1.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

{firecrawl-2.0.1 → firecrawl-2.1.0}/firecrawl/firecrawl.py

@@ -27,7 +27,7 @@ from pydantic import Field
 # Suppress Pydantic warnings about attribute shadowing
 warnings.filterwarnings("ignore", message="Field name \"json\" in \"FirecrawlDocument\" shadows an attribute in parent \"BaseModel\"")
 warnings.filterwarnings("ignore", message="Field name \"json\" in \"ChangeTrackingData\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ExtractConfig\" shadows an attribute in parent \"BaseModel\"")
+warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonConfig\" shadows an attribute in parent \"BaseModel\"")
 warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ExtractParams\" shadows an attribute in parent \"BaseModel\"")
 
 
@@ -84,7 +84,6 @@ T = TypeVar('T')
 #     statusCode: Optional[int] = None
 #     error: Optional[str] = None
 
-
 class AgentOptions(pydantic.BaseModel):
     """Configuration for the agent."""
     model: Literal["FIRE-1"] = "FIRE-1"
@@ -98,6 +97,16 @@ class ActionsResult(pydantic.BaseModel):
     """Result of actions performed during scraping."""
     screenshots: List[str]
 
+class ChangeTrackingData(pydantic.BaseModel):
+    """
+    Data for the change tracking format.
+    """
+    previousScrapeAt: Optional[str] = None
+    changeStatus: str  # "new" | "same" | "changed" | "removed"
+    visibility: str  # "visible" | "hidden"
+    diff: Optional[Dict[str, Any]] = None
+    json: Optional[Any] = None
+
 class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
     """Document retrieved or processed by Firecrawl."""
     url: Optional[str] = None
@@ -112,6 +121,7 @@ class FirecrawlDocument(pydantic.BaseModel, Generic[T]):
     actions: Optional[ActionsResult] = None
     title: Optional[str] = None  # v1 search only
     description: Optional[str] = None  # v1 search only
+    changeTracking: Optional[ChangeTrackingData] = None
 
 class LocationConfig(pydantic.BaseModel):
     """Location configuration for scraping."""
@@ -125,9 +135,9 @@ class WebhookConfig(pydantic.BaseModel):
     metadata: Optional[Dict[str, str]] = None
     events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None
 
-class CommonOptions(pydantic.BaseModel):
+class ScrapeOptions(pydantic.BaseModel):
     """Parameters for scraping operations."""
-    formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None
+    formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None
     headers: Optional[Dict[str, str]] = None
     includeTags: Optional[List[str]] = None
     excludeTags: Optional[List[str]] = None
@@ -187,17 +197,17 @@ class ExtractAgent(pydantic.BaseModel):
     """Configuration for the agent in extract operations."""
     model: Literal["FIRE-1"] = "FIRE-1"
 
-class ExtractConfig(pydantic.BaseModel):
+class JsonConfig(pydantic.BaseModel):
     """Configuration for extraction."""
     prompt: Optional[str] = None
     schema: Optional[Any] = None
     systemPrompt: Optional[str] = None
     agent: Optional[ExtractAgent] = None
 
-class ScrapeParams(CommonOptions):
+class ScrapeParams(ScrapeOptions):
     """Parameters for scraping operations."""
-    extract: Optional[ExtractConfig] = None
-    jsonOptions: Optional[ExtractConfig] = None
+    extract: Optional[JsonConfig] = None
+    jsonOptions: Optional[JsonConfig] = None
     actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None
     agent: Optional[AgentOptions] = None
 
@@ -236,7 +246,7 @@ class CrawlParams(pydantic.BaseModel):
     allowBackwardLinks: Optional[bool] = None
     allowExternalLinks: Optional[bool] = None
     ignoreSitemap: Optional[bool] = None
-    scrapeOptions: Optional[CommonOptions] = None
+    scrapeOptions: Optional[ScrapeOptions] = None
     webhook: Optional[Union[str, WebhookConfig]] = None
     deduplicateSimilarURLs: Optional[bool] = None
     ignoreQueryParameters: Optional[bool] = None
@@ -290,7 +300,7 @@ class ExtractParams(pydantic.BaseModel):
     includeSubdomains: Optional[bool] = None
     origin: Optional[str] = None
     showSources: Optional[bool] = None
-    scrapeOptions: Optional[CommonOptions] = None
+    scrapeOptions: Optional[ScrapeOptions] = None
 
 class ExtractResponse(pydantic.BaseModel, Generic[T]):
     """Response from extract operations."""
@@ -310,7 +320,7 @@ class SearchParams(pydantic.BaseModel):
     location: Optional[str] = None
     origin: Optional[str] = "api"
     timeout: Optional[int] = 60000
-    scrapeOptions: Optional[CommonOptions] = None
+    scrapeOptions: Optional[ScrapeOptions] = None
 
 class SearchResponse(pydantic.BaseModel):
     """Response from search operations."""
@@ -378,16 +388,6 @@ class GenerateLLMsTextStatusResponse(pydantic.BaseModel):
     status: Literal["processing", "completed", "failed"]
     error: Optional[str] = None
     expiresAt: str
-
-class ChangeTrackingData(pydantic.BaseModel):
-    """
-    Data for the change tracking format.
-    """
-    previousScrapeAt: Optional[str] = None
-    changeStatus: str  # "new" | "same" | "changed" | "removed"
-    visibility: str  # "visible" | "hidden"
-    diff: Optional[Dict[str, Any]] = None
-    json: Optional[Any] = None
     
 class SearchResponse(pydantic.BaseModel):
     """
@@ -443,7 +443,7 @@ class FirecrawlApp:
             self,
             url: str,
             *,
-            formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+            formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,
             include_tags: Optional[List[str]] = None,
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
@@ -455,8 +455,8 @@ class FirecrawlApp:
             remove_base64_images: Optional[bool] = None,
             block_ads: Optional[bool] = None,
             proxy: Optional[Literal["basic", "stealth"]] = None,
-            extract: Optional[ExtractConfig] = None,
-            json_options: Optional[ExtractConfig] = None,
+            extract: Optional[JsonConfig] = None,
+            json_options: Optional[JsonConfig] = None,
             actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
             **kwargs) -> ScrapeResponse[Any]:
         """
@@ -476,8 +476,8 @@ class FirecrawlApp:
           remove_base64_images (Optional[bool]): Remove base64 images
           block_ads (Optional[bool]): Block ads
           proxy (Optional[Literal["basic", "stealth"]]): Proxy type (basic/stealth)
-          extract (Optional[ExtractConfig]): Content extraction settings
-          json_options (Optional[ExtractConfig]): JSON extraction settings
+          extract (Optional[JsonConfig]): Content extraction settings
+          json_options (Optional[JsonConfig]): JSON extraction settings
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]]): Actions to perform
 
 
@@ -569,7 +569,7 @@ class FirecrawlApp:
             country: Optional[str] = None,
             location: Optional[str] = None,
             timeout: Optional[int] = None,
-            scrape_options: Optional[CommonOptions] = None,
+            scrape_options: Optional[ScrapeOptions] = None,
             params: Optional[Union[Dict[str, Any], SearchParams]] = None,
             **kwargs) -> SearchResponse:
         """
@@ -584,7 +584,7 @@ class FirecrawlApp:
             country (Optional[str]): Country code (default: "us") 
             location (Optional[str]): Geo-targeting
             timeout (Optional[int]): Request timeout in milliseconds
-            scrape_options (Optional[CommonOptions]): Result scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Result scraping configuration
             params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
             **kwargs: Additional keyword arguments for future compatibility
 
@@ -665,7 +665,7 @@ class FirecrawlApp:
         allow_backward_links: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
-        scrape_options: Optional[CommonOptions] = None,
+        scrape_options: Optional[ScrapeOptions] = None,
         webhook: Optional[Union[str, WebhookConfig]] = None,
         deduplicate_similar_urls: Optional[bool] = None,
         ignore_query_parameters: Optional[bool] = None,
@@ -687,7 +687,7 @@ class FirecrawlApp:
             allow_backward_links (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
-            scrape_options (Optional[CommonOptions]): Page scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
             webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
@@ -769,7 +769,7 @@ class FirecrawlApp:
         allow_backward_links: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
-        scrape_options: Optional[CommonOptions] = None,
+        scrape_options: Optional[ScrapeOptions] = None,
         webhook: Optional[Union[str, WebhookConfig]] = None,
         deduplicate_similar_urls: Optional[bool] = None,
         ignore_query_parameters: Optional[bool] = None,
@@ -790,7 +790,7 @@ class FirecrawlApp:
             allow_backward_links (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
-            scrape_options (Optional[CommonOptions]): Page scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
             webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
@@ -1008,7 +1008,7 @@ class FirecrawlApp:
             allow_backward_links: Optional[bool] = None,
             allow_external_links: Optional[bool] = None,
             ignore_sitemap: Optional[bool] = None,
-            scrape_options: Optional[CommonOptions] = None,
+            scrape_options: Optional[ScrapeOptions] = None,
             webhook: Optional[Union[str, WebhookConfig]] = None,
             deduplicate_similar_urls: Optional[bool] = None,
             ignore_query_parameters: Optional[bool] = None,
@@ -1029,7 +1029,7 @@ class FirecrawlApp:
             allow_backward_links (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
-            scrape_options (Optional[CommonOptions]): Page scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
             webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
@@ -1162,8 +1162,8 @@ class FirecrawlApp:
         remove_base64_images: Optional[bool] = None,
         block_ads: Optional[bool] = None,
         proxy: Optional[Literal["basic", "stealth"]] = None,
-        extract: Optional[ExtractConfig] = None,
-        json_options: Optional[ExtractConfig] = None,
+        extract: Optional[JsonConfig] = None,
+        json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
         poll_interval: Optional[int] = 2,
@@ -1188,8 +1188,8 @@ class FirecrawlApp:
             remove_base64_images (Optional[bool]): Remove base64 encoded images
             block_ads (Optional[bool]): Block advertisements
             proxy (Optional[Literal]): Proxy type to use
-            extract (Optional[ExtractConfig]): Content extraction config
-            json_options (Optional[ExtractConfig]): JSON extraction config
+            extract (Optional[JsonConfig]): Content extraction config
+            json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
             poll_interval (Optional[int]): Seconds between status checks (default: 2)
@@ -1286,8 +1286,8 @@ class FirecrawlApp:
         remove_base64_images: Optional[bool] = None,
         block_ads: Optional[bool] = None,
         proxy: Optional[Literal["basic", "stealth"]] = None,
-        extract: Optional[ExtractConfig] = None,
-        json_options: Optional[ExtractConfig] = None,
+        extract: Optional[JsonConfig] = None,
+        json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
         idempotency_key: Optional[str] = None,
@@ -1311,8 +1311,8 @@ class FirecrawlApp:
             remove_base64_images (Optional[bool]): Remove base64 encoded images
             block_ads (Optional[bool]): Block advertisements
             proxy (Optional[Literal]): Proxy type to use
-            extract (Optional[ExtractConfig]): Content extraction config
-            json_options (Optional[ExtractConfig]): JSON extraction config
+            extract (Optional[JsonConfig]): Content extraction config
+            json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
@@ -1408,8 +1408,8 @@ class FirecrawlApp:
         remove_base64_images: Optional[bool] = None,
         block_ads: Optional[bool] = None,
         proxy: Optional[Literal["basic", "stealth"]] = None,
-        extract: Optional[ExtractConfig] = None,
-        json_options: Optional[ExtractConfig] = None,
+        extract: Optional[JsonConfig] = None,
+        json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
         idempotency_key: Optional[str] = None,
@@ -1433,8 +1433,8 @@ class FirecrawlApp:
             remove_base64_images (Optional[bool]): Remove base64 encoded images
             block_ads (Optional[bool]): Block advertisements
             proxy (Optional[Literal]): Proxy type to use
-            extract (Optional[ExtractConfig]): Content extraction config
-            json_options (Optional[ExtractConfig]): JSON extraction config
+            extract (Optional[JsonConfig]): Content extraction config
+            json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
@@ -1742,7 +1742,7 @@ class FirecrawlApp:
 
     def async_extract(
             self,
-            urls: List[str],
+            urls: Optional[List[str]] = None,
             *,
             prompt: Optional[str] = None,
             schema: Optional[Any] = None,
@@ -1750,8 +1750,7 @@ class FirecrawlApp:
             allow_external_links: Optional[bool] = False,
             enable_web_search: Optional[bool] = False,
             show_sources: Optional[bool] = False,
-            agent: Optional[Dict[str, Any]] = None,
-            idempotency_key: Optional[str] = None) -> ExtractResponse[Any]:
+            agent: Optional[Dict[str, Any]] = None) -> ExtractResponse[Any]:
         """
         Initiate an asynchronous extract job.
 
@@ -1775,7 +1774,7 @@ class FirecrawlApp:
         Raises:
             ValueError: If job initiation fails
         """
-        headers = self._prepare_headers(idempotency_key)
+        headers = self._prepare_headers()
         
         schema = schema
         if schema:
@@ -2707,8 +2706,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             remove_base64_images: Optional[bool] = None,
             block_ads: Optional[bool] = None,
             proxy: Optional[Literal["basic", "stealth"]] = None,
-            extract: Optional[ExtractConfig] = None,
-            json_options: Optional[ExtractConfig] = None,
+            extract: Optional[JsonConfig] = None,
+            json_options: Optional[JsonConfig] = None,
             actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None) -> ScrapeResponse[Any]:
         """
         Scrape and extract content from a URL asynchronously.
@@ -2727,8 +2726,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
           remove_base64_images (Optional[bool]): Remove base64 images
           block_ads (Optional[bool]): Block ads
           proxy (Optional[Literal["basic", "stealth"]]): Proxy type (basic/stealth)
-          extract (Optional[ExtractConfig]): Content extraction settings
-          json_options (Optional[ExtractConfig]): JSON extraction settings
+          extract (Optional[JsonConfig]): Content extraction settings
+          json_options (Optional[JsonConfig]): JSON extraction settings
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]]): Actions to perform
 
         Returns:
@@ -2821,8 +2820,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
         remove_base64_images: Optional[bool] = None,
         block_ads: Optional[bool] = None,
         proxy: Optional[Literal["basic", "stealth"]] = None,
-        extract: Optional[ExtractConfig] = None,
-        json_options: Optional[ExtractConfig] = None,
+        extract: Optional[JsonConfig] = None,
+        json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
         poll_interval: Optional[int] = 2,
@@ -2847,8 +2846,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             remove_base64_images (Optional[bool]): Remove base64 encoded images
             block_ads (Optional[bool]): Block advertisements
             proxy (Optional[Literal]): Proxy type to use
-            extract (Optional[ExtractConfig]): Content extraction config
-            json_options (Optional[ExtractConfig]): JSON extraction config
+            extract (Optional[JsonConfig]): Content extraction config
+            json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
             poll_interval (Optional[int]): Seconds between status checks (default: 2)
@@ -2923,9 +2922,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
             headers
         )
 
-        if response.status_code == 200:
+        if response.get('success'):
             try:
-                id = response.json().get('id')
+                id = response.get('id')
             except:
                 raise Exception(f'Failed to parse Firecrawl response as JSON.')
             return self._monitor_job_status(id, headers, poll_interval)
@@ -2950,8 +2949,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
         remove_base64_images: Optional[bool] = None,
         block_ads: Optional[bool] = None,
         proxy: Optional[Literal["basic", "stealth"]] = None,
-        extract: Optional[ExtractConfig] = None,
-        json_options: Optional[ExtractConfig] = None,
+        extract: Optional[JsonConfig] = None,
+        json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
         idempotency_key: Optional[str] = None,
@@ -2975,8 +2974,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             remove_base64_images (Optional[bool]): Remove base64 encoded images
             block_ads (Optional[bool]): Block advertisements
             proxy (Optional[Literal]): Proxy type to use
-            extract (Optional[ExtractConfig]): Content extraction config
-            json_options (Optional[ExtractConfig]): JSON extraction config
+            extract (Optional[JsonConfig]): Content extraction config
+            json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
@@ -3051,7 +3050,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             headers
         )
 
-        if response.status_code == 200:
+        if response.get('status_code') == 200:
             try:
                 return BatchScrapeResponse(**response.json())
             except:
@@ -3060,7 +3059,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             self._handle_error(response, 'start batch scrape job')
 
     async def crawl_url(
-                    self,
+        self,
         url: str,
         *,
         include_paths: Optional[List[str]] = None,
@@ -3071,7 +3070,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         allow_backward_links: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
-        scrape_options: Optional[CommonOptions] = None,
+        scrape_options: Optional[ScrapeOptions] = None,
         webhook: Optional[Union[str, WebhookConfig]] = None,
         deduplicate_similar_urls: Optional[bool] = None,
         ignore_query_parameters: Optional[bool] = None,
@@ -3093,7 +3092,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             allow_backward_links (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
-            scrape_options (Optional[CommonOptions]): Page scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
             webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
@@ -3149,15 +3148,15 @@ class AsyncFirecrawlApp(FirecrawlApp):
         params_dict = final_params.dict(exclude_none=True)
         params_dict['url'] = url
         params_dict['origin'] = f"python-sdk@{version}"
-
         # Make request
         headers = self._prepare_headers(idempotency_key)
         response = await self._async_post_request(
           f'{self.api_url}/v1/crawl', params_dict, headers)
 
-        if response.status_code == 200:
+        print(response)
+        if response.get('success'):
             try:
-                id = response.json().get('id')
+                id = response.get('id')
             except:
                 raise Exception(f'Failed to parse Firecrawl response as JSON.')
             return self._monitor_job_status(id, headers, poll_interval)
@@ -3177,11 +3176,12 @@ class AsyncFirecrawlApp(FirecrawlApp):
         allow_backward_links: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
-        scrape_options: Optional[CommonOptions] = None,
+        scrape_options: Optional[ScrapeOptions] = None,
         webhook: Optional[Union[str, WebhookConfig]] = None,
         deduplicate_similar_urls: Optional[bool] = None,
         ignore_query_parameters: Optional[bool] = None,
         regex_on_full_url: Optional[bool] = None,
+        poll_interval: Optional[int] = 2,
         idempotency_key: Optional[str] = None,
         **kwargs
     ) -> CrawlResponse:
@@ -3198,7 +3198,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             allow_backward_links (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
-            scrape_options (Optional[CommonOptions]): Page scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
             webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
@@ -3263,9 +3263,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
           headers
         )
 
-        if response.status_code == 200:
+        if response.get('success'):
             try:
-                return CrawlResponse(**response.json())
+                return CrawlResponse(**response)
             except:
                 raise Exception(f'Failed to parse Firecrawl response as JSON.')
         else:
@@ -3304,7 +3304,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             headers
         )
 
-        if status_data['status'] == 'completed':
+        if status_data.get('status') == 'completed':
             if 'data' in status_data:
                 data = status_data['data']
                 while 'next' in status_data:
@@ -3318,26 +3318,24 @@ class AsyncFirecrawlApp(FirecrawlApp):
                     data.extend(next_data.get('data', []))
                     status_data = next_data
                 status_data['data'] = data
-
-        response = {
-            'status': status_data.get('status'),
-            'total': status_data.get('total'),
-            'completed': status_data.get('completed'),
-            'creditsUsed': status_data.get('creditsUsed'),
-            'expiresAt': status_data.get('expiresAt'),
-            'data': status_data.get('data')
-        }
+        # Create CrawlStatusResponse object from status data
+        response = CrawlStatusResponse(
+            status=status_data.get('status'),
+            total=status_data.get('total'),
+            completed=status_data.get('completed'),
+            creditsUsed=status_data.get('creditsUsed'),
+            expiresAt=status_data.get('expiresAt'),
+            data=status_data.get('data'),
+            success=False if 'error' in status_data else True
+        )
 
         if 'error' in status_data:
-            response['error'] = status_data['error']
+            response.error = status_data.get('error')
 
         if 'next' in status_data:
-            response['next'] = status_data['next']
+            response.next = status_data.get('next')
 
-        return {
-            'success': False if 'error' in status_data else True,
-            **response
-        }
+        return response
 
     async def _async_monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int = 2) -> CrawlStatusResponse:
         """
@@ -3360,7 +3358,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
                 headers
             )
 
-            if status_data['status'] == 'completed':
+            if status_data.get('status') == 'completed':
                 if 'data' in status_data:
                     data = status_data['data']
                     while 'next' in status_data:
@@ -3377,15 +3375,22 @@ class AsyncFirecrawlApp(FirecrawlApp):
                     return status_data
                 else:
                     raise Exception('Job completed but no data was returned')
-            elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:
+            elif status_data.get('status') in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:
                 await asyncio.sleep(max(poll_interval, 2))
             else:
                 raise Exception(f'Job failed or was stopped. Status: {status_data["status"]}')
 
     async def map_url(
-            self,
-            url: str,
-            params: Optional[MapParams] = None) -> MapResponse:
+        self,
+        url: str,
+        *,
+        search: Optional[str] = None,
+        ignore_sitemap: Optional[bool] = None,
+        include_subdomains: Optional[bool] = None,
+        sitemap_only: Optional[bool] = None,
+        limit: Optional[int] = None,
+        timeout: Optional[int] = None,
+        params: Optional[MapParams] = None) -> MapResponse:
         """
         Asynchronously map and discover links from a URL.
 
@@ -3410,21 +3415,40 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
           Exception: If mapping fails
         """
-        headers = self._prepare_headers()
-        json_data = {'url': url}
+        map_params = {}
         if params:
-            json_data.update(params)
-        json_data['origin'] = f"python-sdk@{version}"
+            map_params.update(params.dict(exclude_none=True))
+
+        # Add individual parameters
+        if search is not None:
+            map_params['search'] = search
+        if ignore_sitemap is not None:
+            map_params['ignoreSitemap'] = ignore_sitemap
+        if include_subdomains is not None:
+            map_params['includeSubdomains'] = include_subdomains
+        if sitemap_only is not None:
+            map_params['sitemapOnly'] = sitemap_only
+        if limit is not None:
+            map_params['limit'] = limit
+        if timeout is not None:
+            map_params['timeout'] = timeout
+
+        # Create final params object
+        final_params = MapParams(**map_params)
+        params_dict = final_params.dict(exclude_none=True)
+        params_dict['url'] = url
+        params_dict['origin'] = f"python-sdk@{version}"
 
+        # Make request
         endpoint = f'/v1/map'
         response = await self._async_post_request(
             f'{self.api_url}{endpoint}',
-            json_data,
-            headers
+            params_dict,
+            headers={"Authorization": f"Bearer {self.api_key}"}
         )
 
         if response.get('success') and 'links' in response:
-            return response
+            return MapResponse(**response)
         elif 'error' in response:
             raise Exception(f'Failed to map URL. Error: {response["error"]}')
         else:
@@ -3432,27 +3456,28 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
     async def extract(
             self,
-            urls: List[str],
-            params: Optional[ExtractParams] = None) -> ExtractResponse[Any]:
+            urls: Optional[List[str]] = None,
+            *,
+            prompt: Optional[str] = None,
+            schema: Optional[Any] = None,
+            system_prompt: Optional[str] = None,
+            allow_external_links: Optional[bool] = False,
+            enable_web_search: Optional[bool] = False,
+            show_sources: Optional[bool] = False,
+            agent: Optional[Dict[str, Any]] = None) -> ExtractResponse[Any]:
+            
         """
         Asynchronously extract structured information from URLs.
 
         Args:
-            urls (List[str]): URLs to extract from
-            params (Optional[ExtractParams]): See ExtractParams model:
-              Extraction Config:
-              * prompt - Custom extraction prompt
-              * schema - JSON schema/Pydantic model
-              * systemPrompt - System context
-              
-              Behavior Options:
-              * allowExternalLinks - Follow external links
-              * enableWebSearch - Enable web search
-              * includeSubdomains - Include subdomains
-              * showSources - Include source URLs
-              
-              Scraping Options:
-              * scrapeOptions - Page scraping config
+            urls (Optional[List[str]]): URLs to extract from
+            prompt (Optional[str]): Custom extraction prompt
+            schema (Optional[Any]): JSON schema/Pydantic model
+            system_prompt (Optional[str]): System context
+            allow_external_links (Optional[bool]): Follow external links
+            enable_web_search (Optional[bool]): Enable web search
+            show_sources (Optional[bool]): Include source URLs
+            agent (Optional[Dict[str, Any]]): Agent configuration
 
         Returns:
           ExtractResponse with:
@@ -3465,29 +3490,35 @@ class AsyncFirecrawlApp(FirecrawlApp):
         """
         headers = self._prepare_headers()
 
-        if not params or (not params.get('prompt') and not params.get('schema')):
+        if not prompt and not schema:
             raise ValueError("Either prompt or schema is required")
 
-        schema = params.get('schema')
+        if not urls and not prompt:
+            raise ValueError("Either urls or prompt is required")
+
         if schema:
             if hasattr(schema, 'model_json_schema'):
+                # Convert Pydantic model to JSON schema
                 schema = schema.model_json_schema()
+            # Otherwise assume it's already a JSON schema dict
 
         request_data = {
-            'urls': urls,
-            'allowExternalLinks': params.get('allow_external_links', params.get('allowExternalLinks', False)),
-            'enableWebSearch': params.get('enable_web_search', params.get('enableWebSearch', False)),
-            'showSources': params.get('show_sources', params.get('showSources', False)),
+            'urls': urls or [],
+            'allowExternalLinks': allow_external_links,
+            'enableWebSearch': enable_web_search,
+            'showSources': show_sources,
             'schema': schema,
-            'origin': f'python-sdk@{version}'
+            'origin': f'python-sdk@{get_version()}'
         }
 
-        if params.get('prompt'):
-            request_data['prompt'] = params['prompt']
-        if params.get('system_prompt'):
-            request_data['systemPrompt'] = params['system_prompt']
-        elif params.get('systemPrompt'):
-            request_data['systemPrompt'] = params['systemPrompt']
+        # Only add prompt and systemPrompt if they exist
+        if prompt:
+            request_data['prompt'] = prompt
+        if system_prompt:
+            request_data['systemPrompt'] = system_prompt
+            
+        if agent:
+            request_data['agent'] = agent
 
         response = await self._async_post_request(
             f'{self.api_url}/v1/extract',
@@ -3507,7 +3538,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
                 )
 
                 if status_data['status'] == 'completed':
-                    return status_data
+                    return ExtractResponse(**status_data)
                 elif status_data['status'] in ['failed', 'cancelled']:
                     raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')
 
@@ -3563,14 +3594,14 @@ class AsyncFirecrawlApp(FirecrawlApp):
                     status_data = next_data
                 status_data['data'] = data
 
-        response = {
-            'status': status_data.get('status'),
-            'total': status_data.get('total'),
-            'completed': status_data.get('completed'),
-            'creditsUsed': status_data.get('creditsUsed'),
-            'expiresAt': status_data.get('expiresAt'),
-            'data': status_data.get('data')
-        }
+        response = BatchScrapeStatusResponse(
+            status=status_data.get('status'),
+            total=status_data.get('total'),
+            completed=status_data.get('completed'),
+            creditsUsed=status_data.get('creditsUsed'),
+            expiresAt=status_data.get('expiresAt'),
+            data=status_data.get('data')
+        )
 
         if 'error' in status_data:
             response['error'] = status_data['error']
@@ -3690,8 +3721,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             allow_external_links: Optional[bool] = False,
             enable_web_search: Optional[bool] = False,
             show_sources: Optional[bool] = False,
-            agent: Optional[Dict[str, Any]] = None,
-            idempotency_key: Optional[str] = None) -> ExtractResponse[Any]:
+            agent: Optional[Dict[str, Any]] = None) -> ExtractResponse[Any]:
         """
         Initiate an asynchronous extraction job without waiting for completion.
 
@@ -3715,7 +3745,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             ValueError: If job initiation fails
         """
-        headers = self._prepare_headers(idempotency_key)
+        headers = self._prepare_headers()
 
         if not prompt and not schema:
             raise ValueError("Either prompt or schema is required")
@@ -3727,14 +3757,14 @@ class AsyncFirecrawlApp(FirecrawlApp):
             if hasattr(schema, 'model_json_schema'):
                 schema = schema.model_json_schema()
 
-        request_data = {
-            'urls': urls or [],
-            'allowExternalLinks': allow_external_links,
-            'enableWebSearch': enable_web_search,
-            'showSources': show_sources,
-            'schema': schema,
-            'origin': f'python-sdk@{version}'
-        }
+        request_data = ExtractResponse(
+            urls=urls or [],
+            allowExternalLinks=allow_external_links,
+            enableWebSearch=enable_web_search,
+            showSources=show_sources,
+            schema=schema,
+            origin=f'python-sdk@{version}'
+        )
 
         if prompt:
             request_data['prompt'] = prompt
@@ -3811,7 +3841,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
             await asyncio.sleep(2)
 
-        return {'success': False, 'error': 'LLMs.txt generation job terminated unexpectedly'}
+        return GenerateLLMsTextStatusResponse(success=False, error='LLMs.txt generation job terminated unexpectedly')
 
     async def async_generate_llms_text(
             self,
@@ -3846,6 +3876,12 @@ class AsyncFirecrawlApp(FirecrawlApp):
         if experimental_stream is not None:
             params['__experimental_stream'] = experimental_stream
 
+        params = GenerateLLMsTextParams(
+            maxUrls=max_urls,
+            showFullText=show_full_text,
+            __experimental_stream=experimental_stream
+        )
+
         headers = self._prepare_headers()
         json_data = {'url': url, **params.dict(exclude_none=True)}
         json_data['origin'] = f"python-sdk@{version}"
@@ -3982,7 +4018,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
 
             await asyncio.sleep(2)
 
-        return {'success': False, 'error': 'Deep research job terminated unexpectedly'}
+        return DeepResearchStatusResponse(success=False, error='Deep research job terminated unexpectedly')
 
     async def async_deep_research(
             self,
@@ -4089,7 +4125,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             country: Optional[str] = None,
             location: Optional[str] = None,
             timeout: Optional[int] = None,
-            scrape_options: Optional[CommonOptions] = None,
+            scrape_options: Optional[ScrapeOptions] = None,
             params: Optional[Union[Dict[str, Any], SearchParams]] = None,
             **kwargs) -> SearchResponse:
         """
@@ -4104,7 +4140,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             country (Optional[str]): Country code (default: "us") 
             location (Optional[str]): Geo-targeting
             timeout (Optional[int]): Request timeout in milliseconds
-            scrape_options (Optional[CommonOptions]): Result scraping configuration
+            scrape_options (Optional[ScrapeOptions]): Result scraping configuration
             params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
             **kwargs: Additional keyword arguments for future compatibility
 

{firecrawl-2.0.1 → firecrawl-2.1.0}/firecrawl.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 2.0.1
+Version: 2.1.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai