firecrawl 1.3.1__tar.gz → 1.4.0__tar.gz

{firecrawl-1.3.1 → firecrawl-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 1.3.1
+Version: 1.4.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
@@ -189,6 +189,69 @@ async def start_crawl_and_watch():
 await start_crawl_and_watch()
 ```
 
+### Scraping multiple URLs in batch
+
+To batch scrape multiple URLs, use the `batch_scrape_urls` method. It takes the URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper such as the output formats.
+
+```python
+idempotency_key = str(uuid.uuid4()) # optional idempotency key
+batch_scrape_result = app.batch_scrape_urls(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']}, 2, idempotency_key)
+print(batch_scrape_result)
+```
+
+### Asynchronous batch scrape
+
+To run a batch scrape asynchronously, use the `async_batch_scrape_urls` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper, such as the output formats.
+
+```python
+batch_scrape_result = app.async_batch_scrape_urls(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']})
+print(batch_scrape_result)
+```
+
+### Checking batch scrape status
+
+To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
+
+```python
+id = batch_scrape_result['id']
+status = app.check_batch_scrape_job(id)
+```
+
+### Batch scrape with WebSockets
+
+To use batch scrape with WebSockets, use the `batch_scrape_urls_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper, such as the output formats.
+
+```python
+# inside an async function...
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+    print("DOC", detail)
+
+def on_error(detail):
+    print("ERR", detail['error'])
+
+def on_done(detail):
+    print("DONE", detail['status'])
+
+# Function to start the crawl and watch process
+async def start_crawl_and_watch():
+    # Initiate the crawl job and get the watcher
+    watcher = app.batch_scrape_urls_and_watch(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']})
+
+    # Add event listeners
+    watcher.add_event_listener("document", on_document)
+    watcher.add_event_listener("error", on_error)
+    watcher.add_event_listener("done", on_done)
+
+    # Start the watcher
+    await watcher.connect()
+
+# Run the event loop
+await start_crawl_and_watch()
+```
+
 ## Error Handling
 
 The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.

{firecrawl-1.3.1 → firecrawl-1.4.0}/README.md

@@ -149,6 +149,69 @@ async def start_crawl_and_watch():
 await start_crawl_and_watch()
 ```
 
+### Scraping multiple URLs in batch
+
+To batch scrape multiple URLs, use the `batch_scrape_urls` method. It takes the URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper such as the output formats.
+
+```python
+idempotency_key = str(uuid.uuid4()) # optional idempotency key
+batch_scrape_result = app.batch_scrape_urls(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']}, 2, idempotency_key)
+print(batch_scrape_result)
+```
+
+### Asynchronous batch scrape
+
+To run a batch scrape asynchronously, use the `async_batch_scrape_urls` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper, such as the output formats.
+
+```python
+batch_scrape_result = app.async_batch_scrape_urls(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']})
+print(batch_scrape_result)
+```
+
+### Checking batch scrape status
+
+To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
+
+```python
+id = batch_scrape_result['id']
+status = app.check_batch_scrape_job(id)
+```
+
+### Batch scrape with WebSockets
+
+To use batch scrape with WebSockets, use the `batch_scrape_urls_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper, such as the output formats.
+
+```python
+# inside an async function...
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+    print("DOC", detail)
+
+def on_error(detail):
+    print("ERR", detail['error'])
+
+def on_done(detail):
+    print("DONE", detail['status'])
+
+# Function to start the crawl and watch process
+async def start_crawl_and_watch():
+    # Initiate the crawl job and get the watcher
+    watcher = app.batch_scrape_urls_and_watch(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']})
+
+    # Add event listeners
+    watcher.add_event_listener("document", on_document)
+    watcher.add_event_listener("error", on_error)
+    watcher.add_event_listener("done", on_done)
+
+    # Start the watcher
+    await watcher.connect()
+
+# Run the event loop
+await start_crawl_and_watch()
+```
+
 ## Error Handling
 
 The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.

{firecrawl-1.3.1 → firecrawl-1.4.0}/firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp
 
-__version__ = "1.3.1"
+__version__ = "1.4.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

{firecrawl-1.3.1 → firecrawl-1.4.0}/firecrawl/firecrawl.py

@@ -81,8 +81,10 @@ class FirecrawlApp:
             response = response.json()
             if response['success'] and 'data' in response:
                 return response['data']
-            else:
+            elif "error" in response:
                 raise Exception(f'Failed to scrape URL. Error: {response["error"]}')
+            else:
+                raise Exception(f'Failed to scrape URL. Error: {response}')
         else:
             self._handle_error(response, 'scrape URL')
 
@@ -266,11 +268,130 @@ class FirecrawlApp:
             response = response.json()
             if response['success'] and 'links' in response:
                 return response
-            else:
+            elif 'error' in response:
                 raise Exception(f'Failed to map URL. Error: {response["error"]}')
+            else:
+                raise Exception(f'Failed to map URL. Error: {response}')
         else:
             self._handle_error(response, 'map')
 
+    def batch_scrape_urls(self, urls: list[str],
+                  params: Optional[Dict[str, Any]] = None,
+                  poll_interval: Optional[int] = 2,
+                  idempotency_key: Optional[str] = None) -> Any:
+        """
+        Initiate a batch scrape job for the specified URLs using the Firecrawl API.
+
+        Args:
+            urls (list[str]): The URLs to scrape.
+            params (Optional[Dict[str, Any]]): Additional parameters for the scraper.
+            poll_interval (Optional[int]): Time in seconds between status checks when waiting for job completion. Defaults to 2 seconds.
+            idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the scrape results. The structure includes:
+                - 'success' (bool): Indicates if the batch scrape was successful.
+                - 'status' (str): The final status of the batch scrape job (e.g., 'completed').
+                - 'completed' (int): Number of scraped pages that completed.
+                - 'total' (int): Total number of scraped pages.
+                - 'creditsUsed' (int): Estimated number of API credits used for this batch scrape.
+                - 'expiresAt' (str): ISO 8601 formatted date-time string indicating when the batch scrape data expires.
+                - 'data' (List[Dict]): List of all the scraped pages.
+
+        Raises:
+            Exception: If the batch scrape job initiation or monitoring fails.
+        """
+        endpoint = f'/v1/batch/scrape'
+        headers = self._prepare_headers(idempotency_key)
+        json_data = {'urls': urls}
+        if params:
+            json_data.update(params)
+        response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)
+        if response.status_code == 200:
+            id = response.json().get('id')
+            return self._monitor_job_status(id, headers, poll_interval)
+
+        else:
+            self._handle_error(response, 'start batch scrape job')
+
+
+    def async_batch_scrape_urls(self, urls: list[str], params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Initiate a crawl job asynchronously.
+
+        Args:
+            urls (list[str]): The URLs to scrape.
+            params (Optional[Dict[str, Any]]): Additional parameters for the scraper.
+            idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the batch scrape initiation response. The structure includes:
+                - 'success' (bool): Indicates if the batch scrape initiation was successful.
+                - 'id' (str): The unique identifier for the batch scrape job.
+                - 'url' (str): The URL to check the status of the batch scrape job.
+        """
+        endpoint = f'/v1/batch/scrape'
+        headers = self._prepare_headers(idempotency_key)
+        json_data = {'urls': urls}
+        if params:
+            json_data.update(params)
+        response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)
+        if response.status_code == 200:
+            return response.json()
+        else:
+            self._handle_error(response, 'start batch scrape job')
+    
+    def batch_scrape_urls_and_watch(self, urls: list[str], params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> 'CrawlWatcher':
+        """
+        Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket.
+
+        Args:
+            urls (list[str]): The URLs to scrape.
+            params (Optional[Dict[str, Any]]): Additional parameters for the scraper.
+            idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
+
+        Returns:
+            CrawlWatcher: An instance of CrawlWatcher to monitor the batch scrape job.
+        """
+        crawl_response = self.async_batch_scrape_urls(urls, params, idempotency_key)
+        if crawl_response['success'] and 'id' in crawl_response:
+            return CrawlWatcher(crawl_response['id'], self)
+        else:
+            raise Exception("Batch scrape job failed to start")
+    
+    def check_batch_scrape_status(self, id: str) -> Any:
+        """
+        Check the status of a batch scrape job using the Firecrawl API.
+
+        Args:
+            id (str): The ID of the batch scrape job.
+
+        Returns:
+            Any: The status of the batch scrape job.
+
+        Raises:
+            Exception: If the status check request fails.
+        """
+        endpoint = f'/v1/batch/scrape/{id}'
+
+        headers = self._prepare_headers()
+        response = self._get_request(f'{self.api_url}{endpoint}', headers)
+        if response.status_code == 200:
+            data = response.json()
+            return {
+                'success': True,
+                'status': data.get('status'),
+                'total': data.get('total'),
+                'completed': data.get('completed'),
+                'creditsUsed': data.get('creditsUsed'),
+                'expiresAt': data.get('expiresAt'),
+                'next': data.get('next'),
+                'data': data.get('data'),
+                'error': data.get('error')
+            }
+        else:
+            self._handle_error(response, 'check batch scrape status')
+
     def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
         """
         Prepare the headers for API requests.

{firecrawl-1.3.1 → firecrawl-1.4.0}/firecrawl.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 1.3.1
+Version: 1.4.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
@@ -189,6 +189,69 @@ async def start_crawl_and_watch():
 await start_crawl_and_watch()
 ```
 
+### Scraping multiple URLs in batch
+
+To batch scrape multiple URLs, use the `batch_scrape_urls` method. It takes the URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper such as the output formats.
+
+```python
+idempotency_key = str(uuid.uuid4()) # optional idempotency key
+batch_scrape_result = app.batch_scrape_urls(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']}, 2, idempotency_key)
+print(batch_scrape_result)
+```
+
+### Asynchronous batch scrape
+
+To run a batch scrape asynchronously, use the `async_batch_scrape_urls` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper, such as the output formats.
+
+```python
+batch_scrape_result = app.async_batch_scrape_urls(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']})
+print(batch_scrape_result)
+```
+
+### Checking batch scrape status
+
+To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
+
+```python
+id = batch_scrape_result['id']
+status = app.check_batch_scrape_job(id)
+```
+
+### Batch scrape with WebSockets
+
+To use batch scrape with WebSockets, use the `batch_scrape_urls_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the scraper, such as the output formats.
+
+```python
+# inside an async function...
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+    print("DOC", detail)
+
+def on_error(detail):
+    print("ERR", detail['error'])
+
+def on_done(detail):
+    print("DONE", detail['status'])
+
+# Function to start the crawl and watch process
+async def start_crawl_and_watch():
+    # Initiate the crawl job and get the watcher
+    watcher = app.batch_scrape_urls_and_watch(['firecrawl.dev', 'mendable.ai'], {'formats': ['markdown', 'html']})
+
+    # Add event listeners
+    watcher.add_event_listener("document", on_document)
+    watcher.add_event_listener("error", on_error)
+    watcher.add_event_listener("done", on_done)
+
+    # Start the watcher
+    await watcher.connect()
+
+# Run the event loop
+await start_crawl_and_watch()
+```
+
 ## Error Handling
 
 The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.

{firecrawl-1.3.1 → firecrawl-1.4.0}/firecrawl.egg-info/SOURCES.txt

@@ -8,8 +8,4 @@ firecrawl.egg-info/PKG-INFO
 firecrawl.egg-info/SOURCES.txt
 firecrawl.egg-info/dependency_links.txt
 firecrawl.egg-info/requires.txt
-firecrawl.egg-info/top_level.txt
-firecrawl/__tests__/e2e_withAuth/__init__.py
-firecrawl/__tests__/e2e_withAuth/test.py
-firecrawl/__tests__/v1/e2e_withAuth/__init__.py
-firecrawl/__tests__/v1/e2e_withAuth/test.py
+firecrawl.egg-info/top_level.txt

firecrawl-1.4.0/firecrawl.egg-info/top_level.txt

@@ -0,0 +1,3 @@
+build
+dist
+firecrawl

firecrawl-1.3.1/firecrawl/__tests__/e2e_withAuth/test.py

@@ -1,170 +0,0 @@
-import importlib.util
-import pytest
-import time
-import os
-from uuid import uuid4
-from dotenv import load_dotenv
-
-load_dotenv()
-
-API_URL = "http://127.0.0.1:3002"
-ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
-TEST_API_KEY = os.getenv('TEST_API_KEY')
-
-print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
-
-spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
-firecrawl = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(firecrawl)
-FirecrawlApp = firecrawl.FirecrawlApp
-
-def test_no_api_key():
-    with pytest.raises(Exception) as excinfo:
-      invalid_app = FirecrawlApp(api_url=API_URL, version='v0')
-    assert "No API key provided" in str(excinfo.value)
-
-def test_scrape_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.scrape_url('https://firecrawl.dev')
-    assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_blocklisted_url():
-    blocklisted_url = "https://facebook.com/fake-test"
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    with pytest.raises(Exception) as excinfo:
-        app.scrape_url(blocklisted_url)
-    assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
-
-def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert 'content' in response
-    assert "_Roast_" in response['content']
-
-def test_scrape_url_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai')
-    print(response)
-
-    assert response is not None
-    assert 'content' in response
-    assert 'markdown' in response
-    assert 'metadata' in response
-    assert 'html' not in response
-    assert "_Roast_" in response['content']
-
-def test_successful_response_with_valid_api_key_and_include_html():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://roastmywebsite.ai', {'pageOptions': {'includeHtml': True}})
-    assert response is not None
-    assert 'content' in response
-    assert 'markdown' in response
-    assert 'html' in response
-    assert 'metadata' in response
-    assert "_Roast_" in response['content']
-    assert "_Roast_" in response['markdown']
-    assert "<h1" in response['html']
-
-def test_successful_response_for_valid_scrape_with_pdf_file():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
-    assert response is not None
-    assert 'content' in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
-
-def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
-    time.sleep(6)  # wait for 6 seconds
-    assert response is not None
-    assert 'content' in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
-
-def test_crawl_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.crawl_url('https://firecrawl.dev')
-    assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_should_return_error_for_blocklisted_url():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    blocklisted_url = "https://twitter.com/fake-test"
-    with pytest.raises(Exception) as excinfo:
-        app.crawl_url(blocklisted_url)
-    assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
-
-def test_crawl_url_wait_for_completion_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
-    assert response is not None
-    assert len(response) > 0
-    assert 'content' in response[0]
-    assert "_Roast_" in response[0]['content']
-
-def test_crawl_url_with_idempotency_key_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    uniqueIdempotencyKey = str(uuid4())
-    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
-    assert response is not None
-    assert len(response) > 0
-    assert 'content' in response[0]
-    assert "_Roast_" in response[0]['content']
-
-    with pytest.raises(Exception) as excinfo:
-        app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
-    assert "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used" in str(excinfo.value) 
-
-def test_check_crawl_status_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, False)
-    assert response is not None
-    assert 'jobId' in response
-    
-    time.sleep(30)  # wait for 30 seconds
-    status_response = app.check_crawl_status(response['jobId'])
-    assert status_response is not None
-    assert 'status' in status_response
-    assert status_response['status'] == 'completed'
-    assert 'data' in status_response
-    assert len(status_response['data']) > 0
-
-def test_search_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.search("test query")
-    assert response is not None
-    assert 'content' in response[0]
-    assert len(response) > 2
-
-def test_search_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.search("test query")
-    assert "Unexpected error during search: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_llm_extraction():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
-    response = app.scrape_url("https://firecrawl.dev", {
-        'extractorOptions': {
-            'mode': 'llm-extraction',
-            'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
-            'extractionSchema': {
-                'type': 'object',
-                'properties': {
-                    'company_mission': {'type': 'string'},
-                    'supports_sso': {'type': 'boolean'},
-                    'is_open_source': {'type': 'boolean'}
-                },
-                'required': ['company_mission', 'supports_sso', 'is_open_source']
-            }
-        }
-    })
-    assert response is not None
-    assert 'llm_extraction' in response
-    llm_extraction = response['llm_extraction']
-    assert 'company_mission' in llm_extraction
-    assert isinstance(llm_extraction['supports_sso'], bool)
-    assert isinstance(llm_extraction['is_open_source'], bool)

firecrawl-1.3.1/firecrawl/__tests__/v1/e2e_withAuth/test.py

@@ -1,352 +0,0 @@
-import importlib.util
-import pytest
-import time
-import os
-from uuid import uuid4
-from dotenv import load_dotenv
-from datetime import datetime
-
-load_dotenv()
-
-API_URL = "http://127.0.0.1:3002";
-ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
-TEST_API_KEY = os.getenv('TEST_API_KEY')
-
-print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
-
-spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
-firecrawl = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(firecrawl)
-FirecrawlApp = firecrawl.FirecrawlApp
-
-def test_no_api_key():
-    with pytest.raises(Exception) as excinfo:
-      invalid_app = FirecrawlApp(api_url=API_URL)
-    assert "No API key provided" in str(excinfo.value)
-
-def test_scrape_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.scrape_url('https://firecrawl.dev')
-    assert "Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_blocklisted_url():
-    blocklisted_url = "https://facebook.com/fake-test"
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    with pytest.raises(Exception) as excinfo:
-        app.scrape_url(blocklisted_url)
-    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
-
-def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
-    response = app.scrape_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert "_Roast_" in response['markdown']
-    assert "content" not in response
-    assert "html" not in response
-    assert "metadata" in response
-    assert "links" not in response
-    assert "rawHtml" not in response
-
-def test_successful_response_for_valid_scrape():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.scrape_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert 'markdown' in response
-    assert "_Roast_" in response['markdown']
-    assert 'metadata' in response
-    assert 'content' not in response
-    assert 'html' not in response
-    assert 'rawHtml' not in response
-    assert 'screenshot' not in response
-    assert 'links' not in response
-
-def test_successful_response_with_valid_api_key_and_options():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    params = {
-        'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
-        'headers': {'x-key': 'test'},
-        'includeTags': ['h1'],
-        'excludeTags': ['h2'],
-        'onlyMainContent': True,
-        'timeout': 30000,
-        'waitFor': 1000
-    }
-    response = app.scrape_url('https://roastmywebsite.ai', params)
-    assert response is not None
-    assert 'content' not in response
-    assert 'markdown' in response
-    assert 'html' in response
-    assert 'rawHtml' in response
-    assert 'screenshot' in response
-    assert 'links' in response
-    assert "_Roast_" in response['markdown']
-    assert "<h1" in response['html']
-    assert "<h1" in response['rawHtml']
-    assert "https://" in response['screenshot']
-    assert len(response['links']) > 0
-    assert "https://" in response['links'][0]
-    assert 'metadata' in response
-    assert 'title' in response['metadata']
-    assert 'description' in response['metadata']
-    assert 'keywords' in response['metadata']
-    assert 'robots' in response['metadata']
-    assert 'ogTitle' in response['metadata']
-    assert 'ogDescription' in response['metadata']
-    assert 'ogUrl' in response['metadata']
-    assert 'ogImage' in response['metadata']
-    assert 'ogLocaleAlternate' in response['metadata']
-    assert 'ogSiteName' in response['metadata']
-    assert 'sourceURL' in response['metadata']
-    assert 'statusCode' in response['metadata']
-    assert 'pageStatusCode' not in response['metadata']
-    assert 'pageError' not in response['metadata']
-    assert 'error' not in response['metadata']
-    assert response['metadata']['title'] == "Roast My Website"
-    assert response['metadata']['description'] == "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"
-    assert response['metadata']['keywords'] == "Roast My Website,Roast,Website,GitHub,Firecrawl"
-    assert response['metadata']['robots'] == "follow, index"
-    assert response['metadata']['ogTitle'] == "Roast My Website"
-    assert response['metadata']['ogDescription'] == "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"
-    assert response['metadata']['ogUrl'] == "https://www.roastmywebsite.ai"
-    assert response['metadata']['ogImage'] == "https://www.roastmywebsite.ai/og.png"
-    assert response['metadata']['ogLocaleAlternate'] == []
-    assert response['metadata']['ogSiteName'] == "Roast My Website"
-    assert response['metadata']['sourceURL'] == "https://roastmywebsite.ai"
-    assert response['metadata']['statusCode'] == 200
-
-def test_successful_response_for_valid_scrape_with_pdf_file():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
-    assert response is not None
-    assert 'content' not in response
-    assert 'metadata' in response
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']
-
-def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
-    time.sleep(1)  # wait for 1 second
-    assert response is not None
-    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']
-
-def test_crawl_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.crawl_url('https://firecrawl.dev')
-    assert "Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_should_return_error_for_blocklisted_url():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    blocklisted_url = "https://twitter.com/fake-test"
-    with pytest.raises(Exception) as excinfo:
-        app.crawl_url(blocklisted_url)
-    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
-
-def test_crawl_url_wait_for_completion_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30)
-    assert response is not None
-    assert 'total' in response
-    assert response['total'] > 0
-    assert 'creditsUsed' in response
-    assert response['creditsUsed'] > 0
-    assert 'expiresAt' in response
-    assert datetime.strptime(response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
-    assert 'status' in response
-    assert response['status'] == 'completed'
-    assert 'next' not in response
-    assert len(response['data']) > 0
-    assert 'markdown' in response['data'][0]
-    assert "_Roast_" in response['data'][0]['markdown']
-    assert 'content' not in response['data'][0]
-    assert 'html' not in response['data'][0]
-    assert 'rawHtml' not in response['data'][0]
-    assert 'screenshot' not in response['data'][0]
-    assert 'links' not in response['data'][0]
-    assert 'metadata' in response['data'][0]
-    assert 'title' in response['data'][0]['metadata']
-    assert 'description' in response['data'][0]['metadata']
-    assert 'language' in response['data'][0]['metadata']
-    assert 'sourceURL' in response['data'][0]['metadata']
-    assert 'statusCode' in response['data'][0]['metadata']
-    assert 'error' not in response['data'][0]['metadata']
-
-def test_crawl_url_with_options_and_wait_for_completion():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.crawl_url('https://roastmywebsite.ai', {
-        'excludePaths': ['blog/*'],
-        'includePaths': ['/'],
-        'maxDepth': 2,
-        'ignoreSitemap': True,
-        'limit': 10,
-        'allowBackwardLinks': True,
-        'allowExternalLinks': True,
-        'scrapeOptions': {
-            'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
-            'headers': {"x-key": "test"},
-            'includeTags': ['h1'],
-            'excludeTags': ['h2'],
-            'onlyMainContent': True,
-            'waitFor': 1000
-        }
-    }, True, 30)
-    assert response is not None
-    assert 'total' in response
-    assert response['total'] > 0
-    assert 'creditsUsed' in response
-    assert response['creditsUsed'] > 0
-    assert 'expiresAt' in response
-    assert datetime.strptime(response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
-    assert 'status' in response
-    assert response['status'] == 'completed'
-    assert 'next' not in response
-    assert len(response['data']) > 0
-    assert 'markdown' in response['data'][0]
-    assert "_Roast_" in response['data'][0]['markdown']
-    assert 'content' not in response['data'][0]
-    assert 'html' in response['data'][0]
-    assert "<h1" in response['data'][0]['html']
-    assert 'rawHtml' in response['data'][0]
-    assert "<h1" in response['data'][0]['rawHtml']
-    assert 'screenshot' in response['data'][0]
-    assert "https://" in response['data'][0]['screenshot']
-    assert 'links' in response['data'][0]
-    assert len(response['data'][0]['links']) > 0
-    assert 'metadata' in response['data'][0]
-    assert 'title' in response['data'][0]['metadata']
-    assert 'description' in response['data'][0]['metadata']
-    assert 'language' in response['data'][0]['metadata']
-    assert 'sourceURL' in response['data'][0]['metadata']
-    assert 'statusCode' in response['data'][0]['metadata']
-    assert 'error' not in response['data'][0]['metadata']
-
-def test_crawl_url_with_idempotency_key_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    uniqueIdempotencyKey = str(uuid4())
-    response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, False, 2, uniqueIdempotencyKey)
-    assert response is not None
-    assert 'id' in response
-
-    with pytest.raises(Exception) as excinfo:
-        app.crawl_url('https://firecrawl.dev', {'excludePaths': ['blog/*']}, True, 2, uniqueIdempotencyKey)
-    assert "Idempotency key already used" in str(excinfo.value)
-
-def test_check_crawl_status_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.crawl_url('https://firecrawl.dev', {'scrapeOptions': {'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}}, False)
-    assert response is not None
-    assert 'id' in response
-    
-    max_checks = 15
-    checks = 0
-    status_response = app.check_crawl_status(response['id'])
-    
-    while status_response['status'] == 'scraping' and checks < max_checks:
-        time.sleep(1)  # wait for 1 second
-        assert 'partial_data' not in status_response
-        assert 'current' not in status_response
-        assert 'data' in status_response
-        assert 'total' in status_response
-        assert 'creditsUsed' in status_response
-        assert 'expiresAt' in status_response
-        assert 'status' in status_response
-        assert 'next' in status_response
-        assert status_response['total'] > 0
-        assert status_response['creditsUsed'] > 0
-        assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
-        assert status_response['status'] == 'scraping'
-        assert '/v1/crawl/' in status_response['next']
-        status_response = app.check_crawl_status(response['id'])
-        checks += 1
-
-    assert status_response is not None
-    assert 'total' in status_response
-    assert status_response['total'] > 0
-    assert 'creditsUsed' in status_response
-    assert status_response['creditsUsed'] > 0
-    assert 'expiresAt' in status_response
-    assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
-    assert 'status' in status_response
-    assert status_response['status'] == 'completed'
-    assert len(status_response['data']) > 0
-    assert 'markdown' in status_response['data'][0]
-    assert len(status_response['data'][0]['markdown']) > 10
-    assert 'content' not in status_response['data'][0]
-    assert 'html' in status_response['data'][0]
-    assert "<div" in status_response['data'][0]['html']
-    assert 'rawHtml' in status_response['data'][0]
-    assert "<div" in status_response['data'][0]['rawHtml']
-    assert 'screenshot' in status_response['data'][0]
-    assert "https://" in status_response['data'][0]['screenshot']
-    assert 'links' in status_response['data'][0]
-    assert status_response['data'][0]['links'] is not None
-    assert len(status_response['data'][0]['links']) > 0
-    assert 'metadata' in status_response['data'][0]
-    assert 'title' in status_response['data'][0]['metadata']
-    assert 'description' in status_response['data'][0]['metadata']
-    assert 'language' in status_response['data'][0]['metadata']
-    assert 'sourceURL' in status_response['data'][0]['metadata']
-    assert 'statusCode' in status_response['data'][0]['metadata']
-    assert 'error' not in status_response['data'][0]['metadata']
-
-def test_invalid_api_key_on_map():
-    invalid_app = FirecrawlApp(api_key="invalid_api_key", api_url=API_URL)
-    with pytest.raises(Exception) as excinfo:
-        invalid_app.map_url('https://roastmywebsite.ai')
-    assert "Unauthorized: Invalid token" in str(excinfo.value)
-
-def test_blocklisted_url_on_map():
-    app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
-    blocklisted_url = "https://facebook.com/fake-test"
-    with pytest.raises(Exception) as excinfo:
-        app.map_url(blocklisted_url)
-    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
-
-def test_successful_response_with_valid_preview_token_on_map():
-    app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)
-    response = app.map_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert len(response) > 0
-
-def test_successful_response_for_valid_map():
-    app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
-    response = app.map_url('https://roastmywebsite.ai')
-    assert response is not None
-    assert len(response) > 0
-    assert any("https://" in link for link in response)
-    filtered_links = [link for link in response if "roastmywebsite.ai" in link]
-    assert len(filtered_links) > 0
-
-def test_search_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    with pytest.raises(NotImplementedError) as excinfo:
-        app.search("test query")
-    assert "Search is not supported in v1" in str(excinfo.value)
-
-# def test_llm_extraction():
-#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-#     response = app.scrape_url("https://mendable.ai", {
-#         'extractorOptions': {
-#             'mode': 'llm-extraction',
-#             'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
-#             'extractionSchema': {
-#                 'type': 'object',
-#                 'properties': {
-#                     'company_mission': {'type': 'string'},
-#                     'supports_sso': {'type': 'boolean'},
-#                     'is_open_source': {'type': 'boolean'}
-#                 },
-#                 'required': ['company_mission', 'supports_sso', 'is_open_source']
-#             }
-#         }
-#     })
-#     assert response is not None
-#     assert 'llm_extraction' in response
-#     llm_extraction = response['llm_extraction']
-#     assert 'company_mission' in llm_extraction
-#     assert isinstance(llm_extraction['supports_sso'], bool)
-#     assert isinstance(llm_extraction['is_open_source'], bool)
-
-
-    

firecrawl-1.3.1/firecrawl.egg-info/top_level.txt

@@ -1 +0,0 @@
-firecrawl