firecrawl-py 2.8.0__py3-none-any.whl → 2.10.0__py3-none-any.whl

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
 
-__version__ = "2.8.0"
+__version__ = "2.10.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/__tests__/v1/e2e_withAuth/test.py

@@ -437,4 +437,29 @@ def test_search_with_invalid_params():
         app.search("test query", {"invalid_param": "value"})
     assert "ValidationError" in str(e.value)
 
+# def test_scrape_url_with_parse_pdf_true():
+#     if TEST_API_KEY:
+#         app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+#         response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf', parse_pdf=True)
+#         assert response is not None
+#         assert 'markdown' in response
+#         assert len(response['markdown']) > 100
+
+# def test_scrape_url_with_parse_pdf_false():
+#     if TEST_API_KEY:
+#         app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+#         response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf', parse_pdf=False)
+#         assert response is not None
+#         assert 'markdown' in response
+#         assert 'h7uKu14adDL6yGfnGf2qycY5uq8kC3OKCWkPxm' in response['markdown']
+
+# def test_scrape_options_with_parse_pdf():
+#     if TEST_API_KEY:
+#         from firecrawl.firecrawl import ScrapeOptions
+#         app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+#         scrape_options = ScrapeOptions(parsePDF=False, formats=['markdown'])
+#         response = app.search("firecrawl", limit=1, scrape_options=scrape_options)
+#         assert response is not None
+#         assert 'data' in response
+
 

firecrawl/firecrawl.py

@@ -160,6 +160,7 @@ class ScrapeOptions(pydantic.BaseModel):
     changeTrackingOptions: Optional[ChangeTrackingOptions] = None
     maxAge: Optional[int] = None
     storeInCache: Optional[bool] = None
+    parsePDF: Optional[bool] = None
 
 class WaitAction(pydantic.BaseModel):
     """Wait action to perform during scraping."""
@@ -263,6 +264,7 @@ class CrawlParams(pydantic.BaseModel):
     ignoreQueryParameters: Optional[bool] = None
     regexOnFullURL: Optional[bool] = None
     delay: Optional[int] = None  # Delay in seconds between scrapes
+    maxConcurrency: Optional[int] = None
 
 class CrawlResponse(pydantic.BaseModel):
     """Response from crawling operations."""
@@ -464,6 +466,7 @@ class FirecrawlApp:
             remove_base64_images: Optional[bool] = None,
             block_ads: Optional[bool] = None,
             proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+            parse_pdf: Optional[bool] = None,
             extract: Optional[JsonConfig] = None,
             json_options: Optional[JsonConfig] = None,
             actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
@@ -537,6 +540,8 @@ class FirecrawlApp:
             scrape_params['blockAds'] = block_ads
         if proxy:
             scrape_params['proxy'] = proxy
+        if parse_pdf is not None:
+            scrape_params['parsePDF'] = parse_pdf
         if extract is not None:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
@@ -686,6 +691,7 @@ class FirecrawlApp:
         max_discovery_depth: Optional[int] = None,
         limit: Optional[int] = None,
         allow_backward_links: Optional[bool] = None,
+        crawl_entire_domain: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
         scrape_options: Optional[ScrapeOptions] = None,
@@ -694,6 +700,7 @@ class FirecrawlApp:
         ignore_query_parameters: Optional[bool] = None,
         regex_on_full_url: Optional[bool] = None,
         delay: Optional[int] = None,
+        max_concurrency: Optional[int] = None,
         poll_interval: Optional[int] = 2,
         idempotency_key: Optional[str] = None,
         **kwargs
@@ -708,7 +715,8 @@ class FirecrawlApp:
             max_depth (Optional[int]): Maximum crawl depth
             max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
             limit (Optional[int]): Maximum pages to crawl
-            allow_backward_links (Optional[bool]): Follow parent directory links
+            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+            crawl_entire_domain (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
             scrape_options (Optional[ScrapeOptions]): Page scraping configuration
@@ -717,6 +725,7 @@ class FirecrawlApp:
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
             regex_on_full_url (Optional[bool]): Apply regex to full URLs
             delay (Optional[int]): Delay in seconds between scrapes
+            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
             poll_interval (Optional[int]): Seconds between status checks (default: 2)
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
             **kwargs: Additional parameters to pass to the API
@@ -746,7 +755,9 @@ class FirecrawlApp:
             crawl_params['maxDiscoveryDepth'] = max_discovery_depth
         if limit is not None:
             crawl_params['limit'] = limit
-        if allow_backward_links is not None:
+        if crawl_entire_domain is not None:
+            crawl_params['crawlEntireDomain'] = crawl_entire_domain
+        elif allow_backward_links is not None:
             crawl_params['allowBackwardLinks'] = allow_backward_links
         if allow_external_links is not None:
             crawl_params['allowExternalLinks'] = allow_external_links
@@ -764,7 +775,9 @@ class FirecrawlApp:
             crawl_params['regexOnFullURL'] = regex_on_full_url
         if delay is not None:
             crawl_params['delay'] = delay
-
+        if max_concurrency is not None:
+            crawl_params['maxConcurrency'] = max_concurrency
+        
         # Add any additional kwargs
         crawl_params.update(kwargs)
 
@@ -797,6 +810,7 @@ class FirecrawlApp:
         max_discovery_depth: Optional[int] = None,
         limit: Optional[int] = None,
         allow_backward_links: Optional[bool] = None,
+        crawl_entire_domain: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
         scrape_options: Optional[ScrapeOptions] = None,
@@ -818,7 +832,8 @@ class FirecrawlApp:
             max_depth (Optional[int]): Maximum crawl depth
             max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
             limit (Optional[int]): Maximum pages to crawl
-            allow_backward_links (Optional[bool]): Follow parent directory links
+            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+            crawl_entire_domain (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
             scrape_options (Optional[ScrapeOptions]): Page scraping configuration
@@ -826,6 +841,8 @@ class FirecrawlApp:
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
             regex_on_full_url (Optional[bool]): Apply regex to full URLs
+            delay (Optional[int]): Delay in seconds between scrapes
+            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
             **kwargs: Additional parameters to pass to the API
 
@@ -855,7 +872,9 @@ class FirecrawlApp:
             crawl_params['maxDiscoveryDepth'] = max_discovery_depth
         if limit is not None:
             crawl_params['limit'] = limit
-        if allow_backward_links is not None:
+        if crawl_entire_domain is not None:
+            crawl_params['crawlEntireDomain'] = crawl_entire_domain
+        elif allow_backward_links is not None:
             crawl_params['allowBackwardLinks'] = allow_backward_links
         if allow_external_links is not None:
             crawl_params['allowExternalLinks'] = allow_external_links
@@ -873,7 +892,9 @@ class FirecrawlApp:
             crawl_params['regexOnFullURL'] = regex_on_full_url
         if delay is not None:
             crawl_params['delay'] = delay
-
+        if max_concurrency is not None:
+            crawl_params['maxConcurrency'] = max_concurrency
+        
         # Add any additional kwargs
         crawl_params.update(kwargs)
 
@@ -1042,6 +1063,7 @@ class FirecrawlApp:
             max_discovery_depth: Optional[int] = None,
             limit: Optional[int] = None,
             allow_backward_links: Optional[bool] = None,
+            crawl_entire_domain: Optional[bool] = None,
             allow_external_links: Optional[bool] = None,
             ignore_sitemap: Optional[bool] = None,
             scrape_options: Optional[ScrapeOptions] = None,
@@ -1049,6 +1071,8 @@ class FirecrawlApp:
             deduplicate_similar_urls: Optional[bool] = None,
             ignore_query_parameters: Optional[bool] = None,
             regex_on_full_url: Optional[bool] = None,
+            delay: Optional[int] = None,
+            max_concurrency: Optional[int] = None,
             idempotency_key: Optional[str] = None,
             **kwargs
     ) -> 'CrawlWatcher':
@@ -1062,7 +1086,8 @@ class FirecrawlApp:
             max_depth (Optional[int]): Maximum crawl depth
             max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
             limit (Optional[int]): Maximum pages to crawl
-            allow_backward_links (Optional[bool]): Follow parent directory links
+            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+            crawl_entire_domain (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
             scrape_options (Optional[ScrapeOptions]): Page scraping configuration
@@ -1070,6 +1095,8 @@ class FirecrawlApp:
             deduplicate_similar_urls (Optional[bool]): Remove similar URLs
             ignore_query_parameters (Optional[bool]): Ignore URL parameters
             regex_on_full_url (Optional[bool]): Apply regex to full URLs
+            delay (Optional[int]): Delay in seconds between scrapes
+            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
             **kwargs: Additional parameters to pass to the API
 
@@ -1094,6 +1121,8 @@ class FirecrawlApp:
             deduplicate_similar_urls=deduplicate_similar_urls,
             ignore_query_parameters=ignore_query_parameters,
             regex_on_full_url=regex_on_full_url,
+            delay=delay,
+            max_concurrency=max_concurrency,
             idempotency_key=idempotency_key,
             **kwargs
         )
@@ -1210,6 +1239,7 @@ class FirecrawlApp:
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
         poll_interval: Optional[int] = 2,
+        max_concurrency: Optional[int] = None,
         idempotency_key: Optional[str] = None,
         **kwargs
     ) -> BatchScrapeStatusResponse:
@@ -1235,6 +1265,7 @@ class FirecrawlApp:
             json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
+            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
             poll_interval (Optional[int]): Seconds between status checks (default: 2)
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
             **kwargs: Additional parameters to pass to the API
@@ -1294,7 +1325,9 @@ class FirecrawlApp:
             scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]
         if agent is not None:
             scrape_params['agent'] = agent.dict(exclude_none=True)
-
+        if max_concurrency is not None:
+            scrape_params['maxConcurrency'] = max_concurrency
+        
         # Add any additional kwargs
         scrape_params.update(kwargs)
 
@@ -1343,6 +1376,7 @@ class FirecrawlApp:
         json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
+        max_concurrency: Optional[int] = None,
         idempotency_key: Optional[str] = None,
         **kwargs
     ) -> BatchScrapeResponse:
@@ -1368,6 +1402,7 @@ class FirecrawlApp:
             json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
+            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
             **kwargs: Additional parameters to pass to the API
 
@@ -1427,7 +1462,9 @@ class FirecrawlApp:
             scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]
         if agent is not None:
             scrape_params['agent'] = agent.dict(exclude_none=True)
-
+        if max_concurrency is not None:
+            scrape_params['maxConcurrency'] = max_concurrency
+        
         # Add any additional kwargs
         scrape_params.update(kwargs)
 
@@ -1475,6 +1512,7 @@ class FirecrawlApp:
         json_options: Optional[JsonConfig] = None,
         actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
         agent: Optional[AgentOptions] = None,
+        max_concurrency: Optional[int] = None,
         idempotency_key: Optional[str] = None,
         **kwargs
     ) -> 'CrawlWatcher':
@@ -1500,6 +1538,7 @@ class FirecrawlApp:
             json_options (Optional[JsonConfig]): JSON extraction config
             actions (Optional[List[Union]]): Actions to perform
             agent (Optional[AgentOptions]): Agent configuration
+            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
             idempotency_key (Optional[str]): Unique key to prevent duplicate requests
             **kwargs: Additional parameters to pass to the API
 
@@ -1555,7 +1594,9 @@ class FirecrawlApp:
             scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]
         if agent is not None:
             scrape_params['agent'] = agent.dict(exclude_none=True)
-
+        if max_concurrency is not None:
+            scrape_params['maxConcurrency'] = max_concurrency
+        
         # Add any additional kwargs
         scrape_params.update(kwargs)
 
@@ -2784,7 +2825,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             * limit - Maximum pages to crawl
 
             Link Following:
-            * allowBackwardLinks - Follow parent directory links
+            * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead
+            * crawlEntireDomain - Follow parent directory links
             * allowExternalLinks - Follow external domain links  
             * ignoreSitemap - Skip sitemap.xml processing
 
@@ -2866,6 +2908,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
             remove_base64_images: Optional[bool] = None,
             block_ads: Optional[bool] = None,
             proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+            parse_pdf: Optional[bool] = None,
             extract: Optional[JsonConfig] = None,
             json_options: Optional[JsonConfig] = None,
             actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,
@@ -2943,6 +2986,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             scrape_params['blockAds'] = block_ads
         if proxy:
             scrape_params['proxy'] = proxy
+        if parse_pdf is not None:
+            scrape_params['parsePDF'] = parse_pdf
         if extract is not None:
             extract = self._ensure_schema_dict(extract)
             if isinstance(extract, dict) and "schema" in extract:
@@ -3263,6 +3308,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         max_discovery_depth: Optional[int] = None,
         limit: Optional[int] = None,
         allow_backward_links: Optional[bool] = None,
+        crawl_entire_domain: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
         scrape_options: Optional[ScrapeOptions] = None,
@@ -3285,7 +3331,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             max_depth (Optional[int]): Maximum crawl depth
             max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
             limit (Optional[int]): Maximum pages to crawl
-            allow_backward_links (Optional[bool]): Follow parent directory links
+            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+            crawl_entire_domain (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
             scrape_options (Optional[ScrapeOptions]): Page scraping configuration
@@ -3323,7 +3370,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
             crawl_params['maxDiscoveryDepth'] = max_discovery_depth
         if limit is not None:
             crawl_params['limit'] = limit
-        if allow_backward_links is not None:
+        if crawl_entire_domain is not None:
+            crawl_params['crawlEntireDomain'] = crawl_entire_domain
+        elif allow_backward_links is not None:
             crawl_params['allowBackwardLinks'] = allow_backward_links
         if allow_external_links is not None:
             crawl_params['allowExternalLinks'] = allow_external_links
@@ -3375,6 +3424,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
         max_discovery_depth: Optional[int] = None,
         limit: Optional[int] = None,
         allow_backward_links: Optional[bool] = None,
+        crawl_entire_domain: Optional[bool] = None,
         allow_external_links: Optional[bool] = None,
         ignore_sitemap: Optional[bool] = None,
         scrape_options: Optional[ScrapeOptions] = None,
@@ -3397,7 +3447,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
             max_depth (Optional[int]): Maximum crawl depth
             max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
             limit (Optional[int]): Maximum pages to crawl
-            allow_backward_links (Optional[bool]): Follow parent directory links
+            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+            crawl_entire_domain (Optional[bool]): Follow parent directory links
             allow_external_links (Optional[bool]): Follow external domain links
             ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
             scrape_options (Optional[ScrapeOptions]): Page scraping configuration
@@ -3431,7 +3482,9 @@ class AsyncFirecrawlApp(FirecrawlApp):
             crawl_params['maxDiscoveryDepth'] = max_discovery_depth
         if limit is not None:
             crawl_params['limit'] = limit
-        if allow_backward_links is not None:
+        if crawl_entire_domain is not None:
+            crawl_params['crawlEntireDomain'] = crawl_entire_domain
+        elif allow_backward_links is not None:
             crawl_params['allowBackwardLinks'] = allow_backward_links
         if allow_external_links is not None:
             crawl_params['allowExternalLinks'] = allow_external_links

{firecrawl_py-2.8.0.dist-info → firecrawl_py-2.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl-py
-Version: 2.8.0
+Version: 2.10.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

firecrawl_py-2.10.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+firecrawl/__init__.py,sha256=qDOTVOIN0WXrkEEWPqy2UfFzbNDbimvD7HOPhXvTkC4,2613
+firecrawl/firecrawl.py,sha256=Bi7n0U94YJicUYnbjKKOmbkrpWh-kSe1ttPpil3rZl4,193869
+firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
+firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=k9IsEbdTHL9Cu49M4FpnQDEo2rnG6RqwmZAsK_EVJr4,21069
+tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+firecrawl_py-2.10.0.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl_py-2.10.0.dist-info/METADATA,sha256=k_qij9hylsX7bmsCfslPrYl0xaQ1B356E7gqVcqTsa4,7169
+firecrawl_py-2.10.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl_py-2.10.0.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl_py-2.10.0.dist-info/RECORD,,

{firecrawl_py-2.8.0.dist-info → firecrawl_py-2.10.0.dist-info}/top_level.txt

@@ -1,4 +1,2 @@
-build
-dist
 firecrawl
 tests

build/lib/build/lib/build/lib/build/lib/firecrawl/__init__.py

@@ -1,79 +0,0 @@
-"""
-This is the Firecrawl package.
-
-This package provides a Python SDK for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs.
-
-For more information visit https://github.com/firecrawl/
-"""
-
-import logging
-import os
-
-from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
-
-__version__ = "2.8.0"
-
-# Define the logger for the Firecrawl project
-logger: logging.Logger = logging.getLogger("firecrawl")
-
-
-def _configure_logger() -> None:
-    """
-    Configure the firecrawl logger for console output.
-
-    The function attaches a handler for console output with a specific format and date
-    format to the firecrawl logger.
-    """
-    try:
-        # Create the formatter
-        formatter = logging.Formatter(
-            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-
-        # Create the console handler and set the formatter
-        console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
-
-        # Add the console handler to the firecrawl logger
-        logger.addHandler(console_handler)
-    except Exception as e:
-        logger.error("Failed to configure logging: %s", e)
-
-
-def setup_logging() -> None:
-    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    # Check if the firecrawl logger already has a handler
-    if logger.hasHandlers():
-        return # To prevent duplicate logging
-
-    # Check if the FIRECRAWL_LOGGING_LEVEL environment variable is set
-    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
-        # Attach a no-op handler to prevent warnings about no handlers
-        logger.addHandler(logging.NullHandler()) 
-        return
-
-    # Attach the console handler to the firecrawl logger
-    _configure_logger()
-
-    # Set the logging level based on the FIRECRAWL_LOGGING_LEVEL environment variable
-    if env == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif env == "INFO":
-        logger.setLevel(logging.INFO)
-    elif env == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif env == "ERROR":
-        logger.setLevel(logging.ERROR)
-    elif env == "CRITICAL":
-        logger.setLevel(logging.CRITICAL)
-    else:
-        logger.setLevel(logging.INFO)
-        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-
-# Initialize logging configuration when the module is imported
-setup_logging()
-logger.debug("Debugging logger setup")

build/lib/build/lib/build/lib/build/lib/firecrawl/__tests__/e2e_withAuth/test.py

@@ -1,170 +0,0 @@
-import importlib.util
-import pytest
-import time
-import os
-from uuid import uuid4
-from dotenv import load_dotenv
-
-load_dotenv()
-
-API_URL = "http://127.0.0.1:3002"
-ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
-TEST_API_KEY = os.getenv('TEST_API_KEY')
-
-print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
-
-spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
-firecrawl = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(firecrawl)
-FirecrawlApp = firecrawl.FirecrawlApp
-
-def test_no_api_key():
-    with pytest.raises(Exception) as excinfo:
-      invalid_app = FirecrawlApp(api_url=API_URL, version='v0')
-    assert "No API key provided" in str(excinfo.value)
-
-def test_scrape_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.scrape_url('https://firecrawl.dev')
-    assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-# def test_blocklisted_url():
-#     blocklisted_url = "https://facebook.com/fake-test"
-#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-#     with pytest.raises(Exception) as excinfo:
-#         app.scrape_url(blocklisted_url)
-#     assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
-
-def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'), version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert 'content' in response
-    assert "_Roast_" in response['content']
-
-def test_scrape_url_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai')
-    print(response)
-
-    assert response is not None
-    assert 'content' in response
-    assert 'markdown' in response
-    assert 'metadata' in response
-    assert 'html' not in response
-    assert "_Roast_" in response['content']
-
-def test_successful_response_with_valid_api_key_and_include_html():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai', {'pageOptions': {'includeHtml': True}})
-    assert response is not None
-    assert 'content' in response
-    assert 'markdown' in response
-    assert 'html' in response
-    assert 'metadata' in response
-    assert "_Roast_" in response['content']
-    assert "_Roast_" in response['markdown']
-    assert "<h1" in response['html']
-
-def test_successful_response_for_valid_scrape_with_pdf_file():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
-    assert response is not None
-    assert 'content' in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
-
-def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
-    time.sleep(6)  # wait for 6 seconds
-    assert response is not None
-    assert 'content' in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
-
-def test_crawl_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.crawl_url('https://firecrawl.dev')
-    assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-# def test_should_return_error_for_blocklisted_url():
-#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-#     blocklisted_url = "https://twitter.com/fake-test"
-#     with pytest.raises(Exception) as excinfo:
-#         app.crawl_url(blocklisted_url)
-#     assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
-
-def test_crawl_url_wait_for_completion_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
-    assert response is not None
-    assert len(response) > 0
-    assert 'content' in response[0]
-    assert "_Roast_" in response[0]['content']
-
-def test_crawl_url_with_idempotency_key_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    uniqueIdempotencyKey = str(uuid4())
-    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
-    assert response is not None
-    assert len(response) > 0
-    assert 'content' in response[0]
-    assert "_Roast_" in response[0]['content']
-
-    with pytest.raises(Exception) as excinfo:
-        app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
-    assert "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used" in str(excinfo.value) 
-
-def test_check_crawl_status_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, False)
-    assert response is not None
-    assert 'jobId' in response
-    
-    time.sleep(30)  # wait for 30 seconds
-    status_response = app.check_crawl_status(response['jobId'])
-    assert status_response is not None
-    assert 'status' in status_response
-    assert status_response['status'] == 'completed'
-    assert 'data' in status_response
-    assert len(status_response['data']) > 0
-
-def test_search_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.search("test query")
-    assert response is not None
-    assert 'content' in response[0]
-    assert len(response) > 2
-
-def test_search_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.search("test query")
-    assert "Unexpected error during search: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_llm_extraction():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url("https://firecrawl.dev", {
-        'extractorOptions': {
-            'mode': 'llm-extraction',
-            'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
-            'extractionSchema': {
-                'type': 'object',
-                'properties': {
-                    'company_mission': {'type': 'string'},
-                    'supports_sso': {'type': 'boolean'},
-                    'is_open_source': {'type': 'boolean'}
-                },
-                'required': ['company_mission', 'supports_sso', 'is_open_source']
-            }
-        }
-    })
-    assert response is not None
-    assert 'llm_extraction' in response
-    llm_extraction = response['llm_extraction']
-    assert 'company_mission' in llm_extraction
-    assert isinstance(llm_extraction['supports_sso'], bool)
-    assert isinstance(llm_extraction['is_open_source'], bool)