firecrawl 2.1.1__py3-none-any.whl → 2.2.0__py3-none-any.whl

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp, JsonConfig, ScrapeOptions # noqa
 
-__version__ = "2.1.1"
+__version__ = "2.2.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/firecrawl.py

@@ -570,7 +570,6 @@ class FirecrawlApp:
             location: Optional[str] = None,
             timeout: Optional[int] = None,
             scrape_options: Optional[ScrapeOptions] = None,
-            params: Optional[Union[Dict[str, Any], SearchParams]] = None,
             **kwargs) -> SearchResponse:
         """
         Search for content using Firecrawl.
@@ -585,7 +584,6 @@ class FirecrawlApp:
             location (Optional[str]): Geo-targeting
             timeout (Optional[int]): Request timeout in milliseconds
             scrape_options (Optional[ScrapeOptions]): Result scraping configuration
-            params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
             **kwargs: Additional keyword arguments for future compatibility
 
         Returns:
@@ -598,13 +596,11 @@ class FirecrawlApp:
         Raises:
             Exception: If search fails or response cannot be parsed
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "search")
+
         # Build search parameters
         search_params = {}
-        if params:
-            if isinstance(params, dict):
-                search_params.update(params)
-            else:
-                search_params.update(params.dict(exclude_none=True))
 
         # Add individual parameters
         if limit is not None:
@@ -705,6 +701,9 @@ class FirecrawlApp:
         Raises:
             Exception: If crawl fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "crawl_url")
+
         crawl_params = {}
 
         # Add individual parameters
@@ -808,6 +807,9 @@ class FirecrawlApp:
         Raises:
             Exception: If crawl initiation fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "async_crawl_url")
+
         crawl_params = {}
 
         # Add individual parameters
@@ -1076,7 +1078,7 @@ class FirecrawlApp:
             sitemap_only: Optional[bool] = None,
             limit: Optional[int] = None,
             timeout: Optional[int] = None,
-            params: Optional[MapParams] = None) -> MapResponse:
+            **kwargs) -> MapResponse:
         """
         Map and discover links from a URL.
 
@@ -1088,7 +1090,7 @@ class FirecrawlApp:
             sitemap_only (Optional[bool]): Only use sitemap.xml
             limit (Optional[int]): Maximum URLs to return
             timeout (Optional[int]): Request timeout in milliseconds
-            params (Optional[MapParams]): Additional mapping parameters
+            **kwargs: Additional parameters to pass to the API
 
         Returns:
             MapResponse: Response containing:
@@ -1099,10 +1101,11 @@ class FirecrawlApp:
         Raises:
             Exception: If mapping fails or response cannot be parsed
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "map_url")
+
         # Build map parameters
         map_params = {}
-        if params:
-            map_params.update(params.dict(exclude_none=True))
 
         # Add individual parameters
         if search is not None:
@@ -1118,6 +1121,9 @@ class FirecrawlApp:
         if timeout is not None:
             map_params['timeout'] = timeout
 
+        # Add any additional kwargs
+        map_params.update(kwargs)
+
         # Create final params object
         final_params = MapParams(**map_params)
         params_dict = final_params.dict(exclude_none=True)
@@ -1205,6 +1211,9 @@ class FirecrawlApp:
         Raises:
             Exception: If batch scrape fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -1328,6 +1337,9 @@ class FirecrawlApp:
         Raises:
             Exception: If job initiation fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "async_batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -1446,6 +1458,9 @@ class FirecrawlApp:
         Raises:
             Exception: If batch scrape job fails to start
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "batch_scrape_urls_and_watch")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -1849,24 +1864,33 @@ class FirecrawlApp:
             show_full_text=show_full_text,
             experimental_stream=experimental_stream
         )
-        if not response.get('success') or 'id' not in response:
-            return response
+        
+        if not response.success or not response.id:
+            return GenerateLLMsTextStatusResponse(
+                success=False,
+                error='Failed to start LLMs.txt generation',
+                status='failed',
+                expiresAt=''
+            )
 
-        job_id = response['id']
+        job_id = response.id
         while True:
             status = self.check_generate_llms_text_status(job_id)
             
-            if status['status'] == 'completed':
+            if status.status == 'completed':
                 return status
-            elif status['status'] == 'failed':
-                raise Exception(f'LLMs.txt generation failed. Error: {status.get("error")}')
-            elif status['status'] != 'processing':
-                break
+            elif status.status == 'failed':
+                return status
+            elif status.status != 'processing':
+                return GenerateLLMsTextStatusResponse(
+                    success=False,
+                    error='LLMs.txt generation job terminated unexpectedly',
+                    status='failed',
+                    expiresAt=''
+                )
 
             time.sleep(2)  # Polling interval
 
-        return {'success': False, 'error': 'LLMs.txt generation job terminated unexpectedly'}
-
     def async_generate_llms_text(
             self,
             url: str,
@@ -1903,10 +1927,13 @@ class FirecrawlApp:
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
-            response = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)
-            if response.status_code == 200:
+            req = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)
+            response = req.json()
+            print("json_data", json_data)
+            print("response", response)
+            if response.get('success'):
                 try:
-                    return response.json()
+                    return GenerateLLMsTextResponse(**response)
                 except:
                     raise Exception('Failed to parse Firecrawl response as JSON.')
             else:
@@ -1914,7 +1941,10 @@ class FirecrawlApp:
         except Exception as e:
             raise ValueError(str(e))
 
-        return {'success': False, 'error': 'Internal server error'}
+        return GenerateLLMsTextResponse(
+            success=False,
+            error='Internal server error'
+        )
 
     def check_generate_llms_text_status(self, id: str) -> GenerateLLMsTextStatusResponse:
         """
@@ -1941,9 +1971,10 @@ class FirecrawlApp:
             response = self._get_request(f'{self.api_url}/v1/llmstxt/{id}', headers)
             if response.status_code == 200:
                 try:
-                    return response.json()
-                except:
-                    raise Exception('Failed to parse Firecrawl response as JSON.')
+                    json_data = response.json()
+                    return GenerateLLMsTextStatusResponse(**json_data)
+                except Exception as e:
+                    raise Exception(f'Failed to parse Firecrawl response as GenerateLLMsTextStatusResponse: {str(e)}')
             elif response.status_code == 404:
                 raise Exception('LLMs.txt generation job not found')
             else:
@@ -1951,7 +1982,7 @@ class FirecrawlApp:
         except Exception as e:
             raise ValueError(str(e))
 
-        return {'success': False, 'error': 'Internal server error'}
+        return GenerateLLMsTextStatusResponse(success=False, error='Internal server error', status='failed', expiresAt='')
 
     def _prepare_headers(
             self,
@@ -2378,6 +2409,56 @@ class FirecrawlApp:
 
         return {'success': False, 'error': 'Internal server error'}
 
+    def _validate_kwargs(self, kwargs: Dict[str, Any], method_name: str) -> None:
+        """
+        Validate additional keyword arguments before they are passed to the API.
+        This provides early validation before the Pydantic model validation.
+
+        Args:
+            kwargs (Dict[str, Any]): Additional keyword arguments to validate
+            method_name (str): Name of the method these kwargs are for
+
+        Raises:
+            ValueError: If kwargs contain invalid or unsupported parameters
+        """
+        if not kwargs:
+            return
+
+        # Known parameter mappings for each method
+        method_params = {
+            "scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for", 
+                          "timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",
+                          "block_ads", "proxy", "extract", "json_options", "actions"},
+            "search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options"},
+            "crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",
+                         "allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",
+                         "webhook", "deduplicate_similar_urls", "ignore_query_parameters", "regex_on_full_url"},
+            "map_url": {"search", "ignore_sitemap", "include_subdomains", "sitemap_only", "limit", "timeout"},
+            "batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+                                 "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+                                 "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+                                 "actions", "agent"},
+            "async_batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+                                       "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+                                       "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+                                       "actions", "agent"},
+            "batch_scrape_urls_and_watch": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+                                           "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+                                           "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+                                           "actions", "agent"}
+        }
+
+        # Get allowed parameters for this method
+        allowed_params = method_params.get(method_name, set())
+        
+        # Check for unknown parameters
+        unknown_params = set(kwargs.keys()) - allowed_params
+        if unknown_params:
+            raise ValueError(f"Unsupported parameter(s) for {method_name}: {', '.join(unknown_params)}. Please refer to the API documentation for the correct parameters.")
+
+        # Additional type validation can be added here if needed
+        # For now, we rely on Pydantic models for detailed type validation
+
 class CrawlWatcher:
     """
     A class to watch and handle crawl job events via WebSocket connection.
@@ -2694,7 +2775,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
     async def scrape_url(
             self,
             url: str,
-            formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+            *,
+            formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,
             include_tags: Optional[List[str]] = None,
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
@@ -2708,9 +2790,10 @@ class AsyncFirecrawlApp(FirecrawlApp):
             proxy: Optional[Literal["basic", "stealth"]] = None,
             extract: Optional[JsonConfig] = None,
             json_options: Optional[JsonConfig] = None,
-            actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None) -> ScrapeResponse[Any]:
+            actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
+            **kwargs) -> ScrapeResponse[Any]:
         """
-        Scrape and extract content from a URL asynchronously.
+        Scrape a single URL asynchronously.
 
         Args:
           url (str): Target URL to scrape
@@ -2729,17 +2812,26 @@ class AsyncFirecrawlApp(FirecrawlApp):
           extract (Optional[JsonConfig]): Content extraction settings
           json_options (Optional[JsonConfig]): JSON extraction settings
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]]): Actions to perform
+          **kwargs: Additional parameters to pass to the API
 
         Returns:
-          ScrapeResponse with:
-          * Requested content formats
-          * Page metadata
-          * Extraction results
-          * Success/error status
+            ScrapeResponse with:
+            * success - Whether scrape was successful
+            * markdown - Markdown content if requested
+            * html - HTML content if requested
+            * rawHtml - Raw HTML content if requested
+            * links - Extracted links if requested
+            * screenshot - Screenshot if requested
+            * extract - Extracted data if requested
+            * json - JSON data if requested
+            * error - Error message if scrape failed
 
         Raises:
-          Exception: If scraping fails
+            Exception: If scraping fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "scrape_url")
+
         headers = self._prepare_headers()
 
         # Build scrape parameters
@@ -2863,6 +2955,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             Exception: If batch scrape fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -2991,6 +3086,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             Exception: If job initiation fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "async_batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -3110,6 +3208,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             Exception: If crawl fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "crawl_url")
+
         crawl_params = {}
 
         # Add individual parameters

{firecrawl-2.1.1.dist-info → firecrawl-2.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 2.1.1
+Version: 2.2.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

{firecrawl-2.1.1.dist-info → firecrawl-2.2.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
-firecrawl/__init__.py,sha256=NU9Qcom12t48ym3ovFMpCYI4-uH-Ac1jnddqSUzxEIE,2570
-firecrawl/firecrawl.py,sha256=bXjJKt2UAdszpoCspBOPen_2lz5ysmVWP5vDMZUbyUo,177726
+firecrawl/__init__.py,sha256=FlLNQdG6xpLs8ppLhPSF-bLx9L3o_A8gmU2UzAACSv8,2570
+firecrawl/firecrawl.py,sha256=wyxYLkEKiW9GO4PKNElewsOJOJzBq_hHIDTl3nd5j94,182693
 firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
 firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=DcCw-cohtnL-t9XPekUtRoQrgg3UCWu8Ikqudf9ory8,19880
 tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
-firecrawl-2.1.1.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
-firecrawl-2.1.1.dist-info/METADATA,sha256=5_5qIPtR-xSv8jAkZLqBP1i-xefxucWl3rZo2OfPsLo,10583
-firecrawl-2.1.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-firecrawl-2.1.1.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
-firecrawl-2.1.1.dist-info/RECORD,,
+firecrawl-2.2.0.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl-2.2.0.dist-info/METADATA,sha256=b4f32c9eyvFDwj11eOUf0KKy1fCsZTKfJFN-eRH_ohA,10583
+firecrawl-2.2.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl-2.2.0.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl-2.2.0.dist-info/RECORD,,