firecrawl 2.1.2__py3-none-any.whl → 2.3.0__py3-none-any.whl

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp, JsonConfig, ScrapeOptions # noqa
 
-__version__ = "2.1.2"
+__version__ = "2.3.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/firecrawl.py

@@ -570,7 +570,6 @@ class FirecrawlApp:
             location: Optional[str] = None,
             timeout: Optional[int] = None,
             scrape_options: Optional[ScrapeOptions] = None,
-            params: Optional[Union[Dict[str, Any], SearchParams]] = None,
             **kwargs) -> SearchResponse:
         """
         Search for content using Firecrawl.
@@ -585,7 +584,6 @@ class FirecrawlApp:
             location (Optional[str]): Geo-targeting
             timeout (Optional[int]): Request timeout in milliseconds
             scrape_options (Optional[ScrapeOptions]): Result scraping configuration
-            params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
             **kwargs: Additional keyword arguments for future compatibility
 
         Returns:
@@ -598,13 +596,11 @@ class FirecrawlApp:
         Raises:
             Exception: If search fails or response cannot be parsed
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "search")
+
         # Build search parameters
         search_params = {}
-        if params:
-            if isinstance(params, dict):
-                search_params.update(params)
-            else:
-                search_params.update(params.dict(exclude_none=True))
 
         # Add individual parameters
         if limit is not None:
@@ -705,6 +701,9 @@ class FirecrawlApp:
         Raises:
             Exception: If crawl fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "crawl_url")
+
         crawl_params = {}
 
         # Add individual parameters
@@ -808,6 +807,9 @@ class FirecrawlApp:
         Raises:
             Exception: If crawl initiation fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "async_crawl_url")
+
         crawl_params = {}
 
         # Add individual parameters
@@ -1076,7 +1078,7 @@ class FirecrawlApp:
             sitemap_only: Optional[bool] = None,
             limit: Optional[int] = None,
             timeout: Optional[int] = None,
-            params: Optional[MapParams] = None) -> MapResponse:
+            **kwargs) -> MapResponse:
         """
         Map and discover links from a URL.
 
@@ -1088,7 +1090,7 @@ class FirecrawlApp:
             sitemap_only (Optional[bool]): Only use sitemap.xml
             limit (Optional[int]): Maximum URLs to return
             timeout (Optional[int]): Request timeout in milliseconds
-            params (Optional[MapParams]): Additional mapping parameters
+            **kwargs: Additional parameters to pass to the API
 
         Returns:
             MapResponse: Response containing:
@@ -1099,10 +1101,11 @@ class FirecrawlApp:
         Raises:
             Exception: If mapping fails or response cannot be parsed
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "map_url")
+
         # Build map parameters
         map_params = {}
-        if params:
-            map_params.update(params.dict(exclude_none=True))
 
         # Add individual parameters
         if search is not None:
@@ -1118,6 +1121,9 @@ class FirecrawlApp:
         if timeout is not None:
             map_params['timeout'] = timeout
 
+        # Add any additional kwargs
+        map_params.update(kwargs)
+
         # Create final params object
         final_params = MapParams(**map_params)
         params_dict = final_params.dict(exclude_none=True)
@@ -1205,6 +1211,9 @@ class FirecrawlApp:
         Raises:
             Exception: If batch scrape fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -1328,6 +1337,9 @@ class FirecrawlApp:
         Raises:
             Exception: If job initiation fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "async_batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -1446,6 +1458,9 @@ class FirecrawlApp:
         Raises:
             Exception: If batch scrape job fails to start
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "batch_scrape_urls_and_watch")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -2394,6 +2409,56 @@ class FirecrawlApp:
 
         return {'success': False, 'error': 'Internal server error'}
 
+    def _validate_kwargs(self, kwargs: Dict[str, Any], method_name: str) -> None:
+        """
+        Validate additional keyword arguments before they are passed to the API.
+        This provides early validation before the Pydantic model validation.
+
+        Args:
+            kwargs (Dict[str, Any]): Additional keyword arguments to validate
+            method_name (str): Name of the method these kwargs are for
+
+        Raises:
+            ValueError: If kwargs contain invalid or unsupported parameters
+        """
+        if not kwargs:
+            return
+
+        # Known parameter mappings for each method
+        method_params = {
+            "scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for", 
+                          "timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",
+                          "block_ads", "proxy", "extract", "json_options", "actions"},
+            "search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options"},
+            "crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",
+                         "allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",
+                         "webhook", "deduplicate_similar_urls", "ignore_query_parameters", "regex_on_full_url"},
+            "map_url": {"search", "ignore_sitemap", "include_subdomains", "sitemap_only", "limit", "timeout"},
+            "batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+                                 "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+                                 "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+                                 "actions", "agent"},
+            "async_batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+                                       "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+                                       "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+                                       "actions", "agent"},
+            "batch_scrape_urls_and_watch": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+                                           "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+                                           "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+                                           "actions", "agent"}
+        }
+
+        # Get allowed parameters for this method
+        allowed_params = method_params.get(method_name, set())
+        
+        # Check for unknown parameters
+        unknown_params = set(kwargs.keys()) - allowed_params
+        if unknown_params:
+            raise ValueError(f"Unsupported parameter(s) for {method_name}: {', '.join(unknown_params)}. Please refer to the API documentation for the correct parameters.")
+
+        # Additional type validation can be added here if needed
+        # For now, we rely on Pydantic models for detailed type validation
+
 class CrawlWatcher:
     """
     A class to watch and handle crawl job events via WebSocket connection.
@@ -2710,7 +2775,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
     async def scrape_url(
             self,
             url: str,
-            formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+            *,
+            formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,
             include_tags: Optional[List[str]] = None,
             exclude_tags: Optional[List[str]] = None,
             only_main_content: Optional[bool] = None,
@@ -2724,9 +2790,10 @@ class AsyncFirecrawlApp(FirecrawlApp):
             proxy: Optional[Literal["basic", "stealth"]] = None,
             extract: Optional[JsonConfig] = None,
             json_options: Optional[JsonConfig] = None,
-            actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None) -> ScrapeResponse[Any]:
+            actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
+            **kwargs) -> ScrapeResponse[Any]:
         """
-        Scrape and extract content from a URL asynchronously.
+        Scrape a single URL asynchronously.
 
         Args:
           url (str): Target URL to scrape
@@ -2745,17 +2812,26 @@ class AsyncFirecrawlApp(FirecrawlApp):
           extract (Optional[JsonConfig]): Content extraction settings
           json_options (Optional[JsonConfig]): JSON extraction settings
           actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]]): Actions to perform
+          **kwargs: Additional parameters to pass to the API
 
         Returns:
-          ScrapeResponse with:
-          * Requested content formats
-          * Page metadata
-          * Extraction results
-          * Success/error status
+            ScrapeResponse with:
+            * success - Whether scrape was successful
+            * markdown - Markdown content if requested
+            * html - HTML content if requested
+            * rawHtml - Raw HTML content if requested
+            * links - Extracted links if requested
+            * screenshot - Screenshot if requested
+            * extract - Extracted data if requested
+            * json - JSON data if requested
+            * error - Error message if scrape failed
 
         Raises:
-          Exception: If scraping fails
+            Exception: If scraping fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "scrape_url")
+
         headers = self._prepare_headers()
 
         # Build scrape parameters
@@ -2879,6 +2955,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             Exception: If batch scrape fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -2943,7 +3022,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
                 id = response.get('id')
             except:
                 raise Exception(f'Failed to parse Firecrawl response as JSON.')
-            return self._monitor_job_status(id, headers, poll_interval)
+            return await self._async_monitor_job_status(id, headers, poll_interval)
         else:
             self._handle_error(response, 'start batch scrape job')
 
@@ -3007,6 +3086,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             Exception: If job initiation fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "async_batch_scrape_urls")
+
         scrape_params = {}
 
         # Add individual parameters
@@ -3126,6 +3208,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
         Raises:
             Exception: If crawl fails
         """
+        # Validate any additional kwargs
+        self._validate_kwargs(kwargs, "crawl_url")
+
         crawl_params = {}
 
         # Add individual parameters
@@ -3174,7 +3259,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
                 id = response.get('id')
             except:
                 raise Exception(f'Failed to parse Firecrawl response as JSON.')
-            return self._monitor_job_status(id, headers, poll_interval)
+            return await self._async_monitor_job_status(id, headers, poll_interval)
         else:
             self._handle_error(response, 'start crawl job')
 
@@ -3387,7 +3472,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
                         data.extend(next_data.get('data', []))
                         status_data = next_data
                     status_data['data'] = data
-                    return status_data
+                    return CrawlStatusResponse(**status_data)
                 else:
                     raise Exception('Job completed but no data was returned')
             elif status_data.get('status') in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:

{firecrawl-2.1.2.dist-info → firecrawl-2.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 2.1.2
+Version: 2.3.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

firecrawl-2.3.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+firecrawl/__init__.py,sha256=t_Vo2P4crdtw85NYeVcgcaQE9VEFxQsKwXQlo2_bjmo,2570
+firecrawl/firecrawl.py,sha256=w7jWQP9i40Je24TvFm7OosYTtml7DffHOHMxr4iaPSM,182740
+firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
+firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=DcCw-cohtnL-t9XPekUtRoQrgg3UCWu8Ikqudf9ory8,19880
+tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+firecrawl-2.3.0.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl-2.3.0.dist-info/METADATA,sha256=kOaXaHbVr8qGNxGK0-h2MNZ20_VxqFuX-RZ5zUXcu6M,10583
+firecrawl-2.3.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl-2.3.0.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl-2.3.0.dist-info/RECORD,,

{firecrawl-2.1.2.dist-info → firecrawl-2.3.0.dist-info}/top_level.txt

@@ -1,4 +1,2 @@
-build
-dist
 firecrawl
 tests

build/lib/build/lib/firecrawl/__init__.py

@@ -1,79 +0,0 @@
-"""
-This is the Firecrawl package.
-
-This package provides a Python SDK for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs.
-
-For more information visit https://github.com/firecrawl/
-"""
-
-import logging
-import os
-
-from .firecrawl import FirecrawlApp, JsonConfig, ScrapeOptions # noqa
-
-__version__ = "2.1.2"
-
-# Define the logger for the Firecrawl project
-logger: logging.Logger = logging.getLogger("firecrawl")
-
-
-def _configure_logger() -> None:
-    """
-    Configure the firecrawl logger for console output.
-
-    The function attaches a handler for console output with a specific format and date
-    format to the firecrawl logger.
-    """
-    try:
-        # Create the formatter
-        formatter = logging.Formatter(
-            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-
-        # Create the console handler and set the formatter
-        console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
-
-        # Add the console handler to the firecrawl logger
-        logger.addHandler(console_handler)
-    except Exception as e:
-        logger.error("Failed to configure logging: %s", e)
-
-
-def setup_logging() -> None:
-    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    # Check if the firecrawl logger already has a handler
-    if logger.hasHandlers():
-        return # To prevent duplicate logging
-
-    # Check if the FIRECRAWL_LOGGING_LEVEL environment variable is set
-    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
-        # Attach a no-op handler to prevent warnings about no handlers
-        logger.addHandler(logging.NullHandler()) 
-        return
-
-    # Attach the console handler to the firecrawl logger
-    _configure_logger()
-
-    # Set the logging level based on the FIRECRAWL_LOGGING_LEVEL environment variable
-    if env == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif env == "INFO":
-        logger.setLevel(logging.INFO)
-    elif env == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif env == "ERROR":
-        logger.setLevel(logging.ERROR)
-    elif env == "CRITICAL":
-        logger.setLevel(logging.CRITICAL)
-    else:
-        logger.setLevel(logging.INFO)
-        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-
-# Initialize logging configuration when the module is imported
-setup_logging()
-logger.debug("Debugging logger setup")

build/lib/build/lib/firecrawl/__tests__/e2e_withAuth/test.py

@@ -1,170 +0,0 @@
-import importlib.util
-import pytest
-import time
-import os
-from uuid import uuid4
-from dotenv import load_dotenv
-
-load_dotenv()
-
-API_URL = "http://127.0.0.1:3002"
-ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
-TEST_API_KEY = os.getenv('TEST_API_KEY')
-
-print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
-
-spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
-firecrawl = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(firecrawl)
-FirecrawlApp = firecrawl.FirecrawlApp
-
-def test_no_api_key():
-    with pytest.raises(Exception) as excinfo:
-      invalid_app = FirecrawlApp(api_url=API_URL, version='v0')
-    assert "No API key provided" in str(excinfo.value)
-
-def test_scrape_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.scrape_url('https://firecrawl.dev')
-    assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-# def test_blocklisted_url():
-#     blocklisted_url = "https://facebook.com/fake-test"
-#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-#     with pytest.raises(Exception) as excinfo:
-#         app.scrape_url(blocklisted_url)
-#     assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
-
-def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'), version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert 'content' in response
-    assert "_Roast_" in response['content']
-
-def test_scrape_url_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai')
-    print(response)
-
-    assert response is not None
-    assert 'content' in response
-    assert 'markdown' in response
-    assert 'metadata' in response
-    assert 'html' not in response
-    assert "_Roast_" in response['content']
-
-def test_successful_response_with_valid_api_key_and_include_html():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai', {'pageOptions': {'includeHtml': True}})
-    assert response is not None
-    assert 'content' in response
-    assert 'markdown' in response
-    assert 'html' in response
-    assert 'metadata' in response
-    assert "_Roast_" in response['content']
-    assert "_Roast_" in response['markdown']
-    assert "<h1" in response['html']
-
-def test_successful_response_for_valid_scrape_with_pdf_file():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
-    assert response is not None
-    assert 'content' in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
-
-def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
-    time.sleep(6)  # wait for 6 seconds
-    assert response is not None
-    assert 'content' in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
-
-def test_crawl_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.crawl_url('https://firecrawl.dev')
-    assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-# def test_should_return_error_for_blocklisted_url():
-#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-#     blocklisted_url = "https://twitter.com/fake-test"
-#     with pytest.raises(Exception) as excinfo:
-#         app.crawl_url(blocklisted_url)
-#     assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
-
-def test_crawl_url_wait_for_completion_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
-    assert response is not None
-    assert len(response) > 0
-    assert 'content' in response[0]
-    assert "_Roast_" in response[0]['content']
-
-def test_crawl_url_with_idempotency_key_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    uniqueIdempotencyKey = str(uuid4())
-    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
-    assert response is not None
-    assert len(response) > 0
-    assert 'content' in response[0]
-    assert "_Roast_" in response[0]['content']
-
-    with pytest.raises(Exception) as excinfo:
-        app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
-    assert "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used" in str(excinfo.value) 
-
-def test_check_crawl_status_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, False)
-    assert response is not None
-    assert 'jobId' in response
-    
-    time.sleep(30)  # wait for 30 seconds
-    status_response = app.check_crawl_status(response['jobId'])
-    assert status_response is not None
-    assert 'status' in status_response
-    assert status_response['status'] == 'completed'
-    assert 'data' in status_response
-    assert len(status_response['data']) > 0
-
-def test_search_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.search("test query")
-    assert response is not None
-    assert 'content' in response[0]
-    assert len(response) > 2
-
-def test_search_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.search("test query")
-    assert "Unexpected error during search: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_llm_extraction():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url("https://firecrawl.dev", {
-        'extractorOptions': {
-            'mode': 'llm-extraction',
-            'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
-            'extractionSchema': {
-                'type': 'object',
-                'properties': {
-                    'company_mission': {'type': 'string'},
-                    'supports_sso': {'type': 'boolean'},
-                    'is_open_source': {'type': 'boolean'}
-                },
-                'required': ['company_mission', 'supports_sso', 'is_open_source']
-            }
-        }
-    })
-    assert response is not None
-    assert 'llm_extraction' in response
-    llm_extraction = response['llm_extraction']
-    assert 'company_mission' in llm_extraction
-    assert isinstance(llm_extraction['supports_sso'], bool)
-    assert isinstance(llm_extraction['is_open_source'], bool)