firecrawl 1.4.0__tar.gz → 1.5.0__tar.gz

{firecrawl-1.4.0 → firecrawl-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 1.4.0
+Version: 1.5.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
@@ -210,11 +210,11 @@ print(batch_scrape_result)
 
 ### Checking batch scrape status
 
-To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
+To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_status` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
 
 ```python
 id = batch_scrape_result['id']
-status = app.check_batch_scrape_job(id)
+status = app.check_batch_scrape_status(id)
 ```
 
 ### Batch scrape with WebSockets

{firecrawl-1.4.0 → firecrawl-1.5.0}/README.md

@@ -170,11 +170,11 @@ print(batch_scrape_result)
 
 ### Checking batch scrape status
 
-To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
+To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_status` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
 
 ```python
 id = batch_scrape_result['id']
-status = app.check_batch_scrape_job(id)
+status = app.check_batch_scrape_status(id)
 ```
 
 ### Batch scrape with WebSockets

firecrawl-1.5.0/firecrawl/__init__.py

@@ -0,0 +1,79 @@
+"""
+This is the Firecrawl package.
+
+This package provides a Python SDK for interacting with the Firecrawl API.
+It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
+and check the status of these jobs.
+
+For more information visit https://github.com/firecrawl/
+"""
+
+import logging
+import os
+
+from .firecrawl import FirecrawlApp # noqa
+
+__version__ = "1.5.0"
+
+# Define the logger for the Firecrawl project
+logger: logging.Logger = logging.getLogger("firecrawl")
+
+
+def _configure_logger() -> None:
+    """
+    Configure the firecrawl logger for console output.
+
+    The function attaches a handler for console output with a specific format and date
+    format to the firecrawl logger.
+    """
+    try:
+        # Create the formatter
+        formatter = logging.Formatter(
+            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+        # Create the console handler and set the formatter
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+
+        # Add the console handler to the firecrawl logger
+        logger.addHandler(console_handler)
+    except Exception as e:
+        logger.error("Failed to configure logging: %s", e)
+
+
+def setup_logging() -> None:
+    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
+    # Check if the firecrawl logger already has a handler
+    if logger.hasHandlers():
+        return # To prevent duplicate logging
+
+    # Check if the FIRECRAWL_LOGGING_LEVEL environment variable is set
+    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
+        # Attach a no-op handler to prevent warnings about no handlers
+        logger.addHandler(logging.NullHandler()) 
+        return
+
+    # Attach the console handler to the firecrawl logger
+    _configure_logger()
+
+    # Set the logging level based on the FIRECRAWL_LOGGING_LEVEL environment variable
+    if env == "DEBUG":
+        logger.setLevel(logging.DEBUG)
+    elif env == "INFO":
+        logger.setLevel(logging.INFO)
+    elif env == "WARNING":
+        logger.setLevel(logging.WARNING)
+    elif env == "ERROR":
+        logger.setLevel(logging.ERROR)
+    elif env == "CRITICAL":
+        logger.setLevel(logging.CRITICAL)
+    else:
+        logger.setLevel(logging.INFO)
+        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
+
+
+# Initialize logging configuration when the module is imported
+setup_logging()
+logger.debug("Debugging logger setup")

firecrawl-1.5.0/firecrawl/__tests__/e2e_withAuth/test.py

@@ -0,0 +1,170 @@
+import importlib.util
+import pytest
+import time
+import os
+from uuid import uuid4
+from dotenv import load_dotenv
+
+load_dotenv()
+
+API_URL = "http://127.0.0.1:3002"
+ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
+TEST_API_KEY = os.getenv('TEST_API_KEY')
+
+print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
+
+spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
+firecrawl = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(firecrawl)
+FirecrawlApp = firecrawl.FirecrawlApp
+
+def test_no_api_key():
+    with pytest.raises(Exception) as excinfo:
+      invalid_app = FirecrawlApp(api_url=API_URL, version='v0')
+    assert "No API key provided" in str(excinfo.value)
+
+def test_scrape_url_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.scrape_url('https://firecrawl.dev')
+    assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_blocklisted_url():
+    blocklisted_url = "https://facebook.com/fake-test"
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    with pytest.raises(Exception) as excinfo:
+        app.scrape_url(blocklisted_url)
+    assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
+
+def test_successful_response_with_valid_preview_token():
+    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')
+    response = app.scrape_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert 'content' in response
+    assert "_Roast_" in response['content']
+
+def test_scrape_url_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.scrape_url('https://roastmywebsite.ai')
+    print(response)
+
+    assert response is not None
+    assert 'content' in response
+    assert 'markdown' in response
+    assert 'metadata' in response
+    assert 'html' not in response
+    assert "_Roast_" in response['content']
+
+def test_successful_response_with_valid_api_key_and_include_html():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.scrape_url('https://roastmywebsite.ai', {'pageOptions': {'includeHtml': True}})
+    assert response is not None
+    assert 'content' in response
+    assert 'markdown' in response
+    assert 'html' in response
+    assert 'metadata' in response
+    assert "_Roast_" in response['content']
+    assert "_Roast_" in response['markdown']
+    assert "<h1" in response['html']
+
+def test_successful_response_for_valid_scrape_with_pdf_file():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
+    assert response is not None
+    assert 'content' in response
+    assert 'metadata' in response
+    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
+
+def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
+    time.sleep(6)  # wait for 6 seconds
+    assert response is not None
+    assert 'content' in response
+    assert 'metadata' in response
+    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
+
+def test_crawl_url_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.crawl_url('https://firecrawl.dev')
+    assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_should_return_error_for_blocklisted_url():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    blocklisted_url = "https://twitter.com/fake-test"
+    with pytest.raises(Exception) as excinfo:
+        app.crawl_url(blocklisted_url)
+    assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
+
+def test_crawl_url_wait_for_completion_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
+    assert response is not None
+    assert len(response) > 0
+    assert 'content' in response[0]
+    assert "_Roast_" in response[0]['content']
+
+def test_crawl_url_with_idempotency_key_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    uniqueIdempotencyKey = str(uuid4())
+    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
+    assert response is not None
+    assert len(response) > 0
+    assert 'content' in response[0]
+    assert "_Roast_" in response[0]['content']
+
+    with pytest.raises(Exception) as excinfo:
+        app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
+    assert "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used" in str(excinfo.value) 
+
+def test_check_crawl_status_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, False)
+    assert response is not None
+    assert 'jobId' in response
+    
+    time.sleep(30)  # wait for 30 seconds
+    status_response = app.check_crawl_status(response['jobId'])
+    assert status_response is not None
+    assert 'status' in status_response
+    assert status_response['status'] == 'completed'
+    assert 'data' in status_response
+    assert len(status_response['data']) > 0
+
+def test_search_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.search("test query")
+    assert response is not None
+    assert 'content' in response[0]
+    assert len(response) > 2
+
+def test_search_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.search("test query")
+    assert "Unexpected error during search: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_llm_extraction():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.scrape_url("https://firecrawl.dev", {
+        'extractorOptions': {
+            'mode': 'llm-extraction',
+            'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+            'extractionSchema': {
+                'type': 'object',
+                'properties': {
+                    'company_mission': {'type': 'string'},
+                    'supports_sso': {'type': 'boolean'},
+                    'is_open_source': {'type': 'boolean'}
+                },
+                'required': ['company_mission', 'supports_sso', 'is_open_source']
+            }
+        }
+    })
+    assert response is not None
+    assert 'llm_extraction' in response
+    llm_extraction = response['llm_extraction']
+    assert 'company_mission' in llm_extraction
+    assert isinstance(llm_extraction['supports_sso'], bool)
+    assert isinstance(llm_extraction['is_open_source'], bool)

firecrawl-1.5.0/firecrawl/__tests__/v1/e2e_withAuth/test.py

@@ -0,0 +1,352 @@
+import importlib.util
+import pytest
+import time
+import os
+from uuid import uuid4
+from dotenv import load_dotenv
+from datetime import datetime
+
+load_dotenv()
+
+API_URL = "http://127.0.0.1:3002";
+ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
+TEST_API_KEY = os.getenv('TEST_API_KEY')
+
+print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
+
+spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
+firecrawl = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(firecrawl)
+FirecrawlApp = firecrawl.FirecrawlApp
+
+def test_no_api_key():
+    with pytest.raises(Exception) as excinfo:
+      invalid_app = FirecrawlApp(api_url=API_URL)
+    assert "No API key provided" in str(excinfo.value)
+
+def test_scrape_url_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.scrape_url('https://firecrawl.dev')
+    assert "Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_blocklisted_url():
+    blocklisted_url = "https://facebook.com/fake-test"
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    with pytest.raises(Exception) as excinfo:
+        app.scrape_url(blocklisted_url)
+    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
+
+def test_successful_response_with_valid_preview_token():
+    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
+    response = app.scrape_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert "_Roast_" in response['markdown']
+    assert "content" not in response
+    assert "html" not in response
+    assert "metadata" in response
+    assert "links" not in response
+    assert "rawHtml" not in response
+
+def test_successful_response_for_valid_scrape():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.scrape_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert 'markdown' in response
+    assert "_Roast_" in response['markdown']
+    assert 'metadata' in response
+    assert 'content' not in response
+    assert 'html' not in response
+    assert 'rawHtml' not in response
+    assert 'screenshot' not in response
+    assert 'links' not in response
+
+def test_successful_response_with_valid_api_key_and_options():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    params = {
+        'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
+        'headers': {'x-key': 'test'},
+        'includeTags': ['h1'],
+        'excludeTags': ['h2'],
+        'onlyMainContent': True,
+        'timeout': 30000,
+        'waitFor': 1000
+    }
+    response = app.scrape_url('https://roastmywebsite.ai', params)
+    assert response is not None
+    assert 'content' not in response
+    assert 'markdown' in response
+    assert 'html' in response
+    assert 'rawHtml' in response
+    assert 'screenshot' in response
+    assert 'links' in response
+    assert "_Roast_" in response['markdown']
+    assert "<h1" in response['html']
+    assert "<h1" in response['rawHtml']
+    assert "https://" in response['screenshot']
+    assert len(response['links']) > 0
+    assert "https://" in response['links'][0]
+    assert 'metadata' in response
+    assert 'title' in response['metadata']
+    assert 'description' in response['metadata']
+    assert 'keywords' in response['metadata']
+    assert 'robots' in response['metadata']
+    assert 'ogTitle' in response['metadata']
+    assert 'ogDescription' in response['metadata']
+    assert 'ogUrl' in response['metadata']
+    assert 'ogImage' in response['metadata']
+    assert 'ogLocaleAlternate' in response['metadata']
+    assert 'ogSiteName' in response['metadata']
+    assert 'sourceURL' in response['metadata']
+    assert 'statusCode' in response['metadata']
+    assert 'pageStatusCode' not in response['metadata']
+    assert 'pageError' not in response['metadata']
+    assert 'error' not in response['metadata']
+    assert response['metadata']['title'] == "Roast My Website"
+    assert response['metadata']['description'] == "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"
+    assert response['metadata']['keywords'] == "Roast My Website,Roast,Website,GitHub,Firecrawl"
+    assert response['metadata']['robots'] == "follow, index"
+    assert response['metadata']['ogTitle'] == "Roast My Website"
+    assert response['metadata']['ogDescription'] == "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"
+    assert response['metadata']['ogUrl'] == "https://www.roastmywebsite.ai"
+    assert response['metadata']['ogImage'] == "https://www.roastmywebsite.ai/og.png"
+    assert response['metadata']['ogLocaleAlternate'] == []
+    assert response['metadata']['ogSiteName'] == "Roast My Website"
+    assert response['metadata']['sourceURL'] == "https://roastmywebsite.ai"
+    assert response['metadata']['statusCode'] == 200
+
+def test_successful_response_for_valid_scrape_with_pdf_file():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
+    assert response is not None
+    assert 'content' not in response
+    assert 'metadata' in response
+    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']
+
+def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
+    time.sleep(1)  # wait for 1 second
+    assert response is not None
+    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']
+
+def test_crawl_url_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.crawl_url('https://firecrawl.dev')
+    assert "Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_should_return_error_for_blocklisted_url():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    blocklisted_url = "https://twitter.com/fake-test"
+    with pytest.raises(Exception) as excinfo:
+        app.crawl_url(blocklisted_url)
+    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
+
+def test_crawl_url_wait_for_completion_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30)
+    assert response is not None
+    assert 'total' in response
+    assert response['total'] > 0
+    assert 'creditsUsed' in response
+    assert response['creditsUsed'] > 0
+    assert 'expiresAt' in response
+    assert datetime.strptime(response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+    assert 'status' in response
+    assert response['status'] == 'completed'
+    assert 'next' not in response
+    assert len(response['data']) > 0
+    assert 'markdown' in response['data'][0]
+    assert "_Roast_" in response['data'][0]['markdown']
+    assert 'content' not in response['data'][0]
+    assert 'html' not in response['data'][0]
+    assert 'rawHtml' not in response['data'][0]
+    assert 'screenshot' not in response['data'][0]
+    assert 'links' not in response['data'][0]
+    assert 'metadata' in response['data'][0]
+    assert 'title' in response['data'][0]['metadata']
+    assert 'description' in response['data'][0]['metadata']
+    assert 'language' in response['data'][0]['metadata']
+    assert 'sourceURL' in response['data'][0]['metadata']
+    assert 'statusCode' in response['data'][0]['metadata']
+    assert 'error' not in response['data'][0]['metadata']
+
+def test_crawl_url_with_options_and_wait_for_completion():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.crawl_url('https://roastmywebsite.ai', {
+        'excludePaths': ['blog/*'],
+        'includePaths': ['/'],
+        'maxDepth': 2,
+        'ignoreSitemap': True,
+        'limit': 10,
+        'allowBackwardLinks': True,
+        'allowExternalLinks': True,
+        'scrapeOptions': {
+            'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
+            'headers': {"x-key": "test"},
+            'includeTags': ['h1'],
+            'excludeTags': ['h2'],
+            'onlyMainContent': True,
+            'waitFor': 1000
+        }
+    }, True, 30)
+    assert response is not None
+    assert 'total' in response
+    assert response['total'] > 0
+    assert 'creditsUsed' in response
+    assert response['creditsUsed'] > 0
+    assert 'expiresAt' in response
+    assert datetime.strptime(response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+    assert 'status' in response
+    assert response['status'] == 'completed'
+    assert 'next' not in response
+    assert len(response['data']) > 0
+    assert 'markdown' in response['data'][0]
+    assert "_Roast_" in response['data'][0]['markdown']
+    assert 'content' not in response['data'][0]
+    assert 'html' in response['data'][0]
+    assert "<h1" in response['data'][0]['html']
+    assert 'rawHtml' in response['data'][0]
+    assert "<h1" in response['data'][0]['rawHtml']
+    assert 'screenshot' in response['data'][0]
+    assert "https://" in response['data'][0]['screenshot']
+    assert 'links' in response['data'][0]
+    assert len(response['data'][0]['links']) > 0
+    assert 'metadata' in response['data'][0]
+    assert 'title' in response['data'][0]['metadata']
+    assert 'description' in response['data'][0]['metadata']
+    assert 'language' in response['data'][0]['metadata']
+    assert 'sourceURL' in response['data'][0]['metadata']
+    assert 'statusCode' in response['data'][0]['metadata']
+    assert 'error' not in response['data'][0]['metadata']
+
+def test_crawl_url_with_idempotency_key_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    uniqueIdempotencyKey = str(uuid4())
+    response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, False, 2, uniqueIdempotencyKey)
+    assert response is not None
+    assert 'id' in response
+
+    with pytest.raises(Exception) as excinfo:
+        app.crawl_url('https://firecrawl.dev', {'excludePaths': ['blog/*']}, True, 2, uniqueIdempotencyKey)
+    assert "Idempotency key already used" in str(excinfo.value)
+
+def test_check_crawl_status_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.crawl_url('https://firecrawl.dev', {'scrapeOptions': {'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}}, False)
+    assert response is not None
+    assert 'id' in response
+    
+    max_checks = 15
+    checks = 0
+    status_response = app.check_crawl_status(response['id'])
+    
+    while status_response['status'] == 'scraping' and checks < max_checks:
+        time.sleep(1)  # wait for 1 second
+        assert 'partial_data' not in status_response
+        assert 'current' not in status_response
+        assert 'data' in status_response
+        assert 'total' in status_response
+        assert 'creditsUsed' in status_response
+        assert 'expiresAt' in status_response
+        assert 'status' in status_response
+        assert 'next' in status_response
+        assert status_response['total'] > 0
+        assert status_response['creditsUsed'] > 0
+        assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+        assert status_response['status'] == 'scraping'
+        assert '/v1/crawl/' in status_response['next']
+        status_response = app.check_crawl_status(response['id'])
+        checks += 1
+
+    assert status_response is not None
+    assert 'total' in status_response
+    assert status_response['total'] > 0
+    assert 'creditsUsed' in status_response
+    assert status_response['creditsUsed'] > 0
+    assert 'expiresAt' in status_response
+    assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+    assert 'status' in status_response
+    assert status_response['status'] == 'completed'
+    assert len(status_response['data']) > 0
+    assert 'markdown' in status_response['data'][0]
+    assert len(status_response['data'][0]['markdown']) > 10
+    assert 'content' not in status_response['data'][0]
+    assert 'html' in status_response['data'][0]
+    assert "<div" in status_response['data'][0]['html']
+    assert 'rawHtml' in status_response['data'][0]
+    assert "<div" in status_response['data'][0]['rawHtml']
+    assert 'screenshot' in status_response['data'][0]
+    assert "https://" in status_response['data'][0]['screenshot']
+    assert 'links' in status_response['data'][0]
+    assert status_response['data'][0]['links'] is not None
+    assert len(status_response['data'][0]['links']) > 0
+    assert 'metadata' in status_response['data'][0]
+    assert 'title' in status_response['data'][0]['metadata']
+    assert 'description' in status_response['data'][0]['metadata']
+    assert 'language' in status_response['data'][0]['metadata']
+    assert 'sourceURL' in status_response['data'][0]['metadata']
+    assert 'statusCode' in status_response['data'][0]['metadata']
+    assert 'error' not in status_response['data'][0]['metadata']
+
+def test_invalid_api_key_on_map():
+    invalid_app = FirecrawlApp(api_key="invalid_api_key", api_url=API_URL)
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.map_url('https://roastmywebsite.ai')
+    assert "Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_blocklisted_url_on_map():
+    app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
+    blocklisted_url = "https://facebook.com/fake-test"
+    with pytest.raises(Exception) as excinfo:
+        app.map_url(blocklisted_url)
+    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
+
+def test_successful_response_with_valid_preview_token_on_map():
+    app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)
+    response = app.map_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert len(response) > 0
+
+def test_successful_response_for_valid_map():
+    app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
+    response = app.map_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert len(response) > 0
+    assert any("https://" in link for link in response)
+    filtered_links = [link for link in response if "roastmywebsite.ai" in link]
+    assert len(filtered_links) > 0
+
+def test_search_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    with pytest.raises(NotImplementedError) as excinfo:
+        app.search("test query")
+    assert "Search is not supported in v1" in str(excinfo.value)
+
+# def test_llm_extraction():
+#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+#     response = app.scrape_url("https://mendable.ai", {
+#         'extractorOptions': {
+#             'mode': 'llm-extraction',
+#             'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+#             'extractionSchema': {
+#                 'type': 'object',
+#                 'properties': {
+#                     'company_mission': {'type': 'string'},
+#                     'supports_sso': {'type': 'boolean'},
+#                     'is_open_source': {'type': 'boolean'}
+#                 },
+#                 'required': ['company_mission', 'supports_sso', 'is_open_source']
+#             }
+#         }
+#     })
+#     assert response is not None
+#     assert 'llm_extraction' in response
+#     llm_extraction = response['llm_extraction']
+#     assert 'company_mission' in llm_extraction
+#     assert isinstance(llm_extraction['supports_sso'], bool)
+#     assert isinstance(llm_extraction['is_open_source'], bool)
+
+
+    

{firecrawl-1.4.0 → firecrawl-1.5.0}/firecrawl/firecrawl.py

@@ -189,17 +189,38 @@ class FirecrawlApp:
         headers = self._prepare_headers()
         response = self._get_request(f'{self.api_url}{endpoint}', headers)
         if response.status_code == 200:
-            data = response.json()
+            status_data = response.json()
+            if status_data['status'] == 'completed':
+                if 'data' in status_data:
+                    data = status_data['data']
+                    while 'next' in status_data:
+                        next_url = status_data.get('next')
+                        if not next_url:
+                            logger.warning("Expected 'next' URL is missing.")
+                            break
+                        try:
+                            status_response = self._get_request(next_url, headers)
+                            if status_response.status_code != 200:
+                                logger.error(f"Failed to fetch next page: {status_response.status_code}")
+                                break
+                            status_data = status_response.json()
+                            data.extend(status_data.get('data', []))
+                        except Exception as e:
+                            logger.error(f"Error during pagination request: {e}")
+                            break
+                        status_data.pop('next', None)
+                    status_data['data'] = data
+                    
             return {
                 'success': True,
-                'status': data.get('status'),
-                'total': data.get('total'),
-                'completed': data.get('completed'),
-                'creditsUsed': data.get('creditsUsed'),
-                'expiresAt': data.get('expiresAt'),
-                'next': data.get('next'),
-                'data': data.get('data'),
-                'error': data.get('error')
+                'status': status_data.get('status'),
+                'total': status_data.get('total'),
+                'completed': status_data.get('completed'),
+                'creditsUsed': status_data.get('creditsUsed'),
+                'expiresAt': status_data.get('expiresAt'),
+                'data': status_data.get('data'),
+                'error': status_data.get('error'),
+                'next': status_data.get('next', None)
             }
         else:
             self._handle_error(response, 'check crawl status')
@@ -377,17 +398,38 @@ class FirecrawlApp:
         headers = self._prepare_headers()
         response = self._get_request(f'{self.api_url}{endpoint}', headers)
         if response.status_code == 200:
-            data = response.json()
+            status_data = response.json()
+            if status_data['status'] == 'completed':
+                if 'data' in status_data:
+                    data = status_data['data']
+                    while 'next' in status_data:
+                        next_url = status_data.get('next')
+                        if not next_url:
+                            logger.warning("Expected 'next' URL is missing.")
+                            break
+                        try:
+                            status_response = self._get_request(next_url, headers)
+                            if status_response.status_code != 200:
+                                logger.error(f"Failed to fetch next page: {status_response.status_code}")
+                                break
+                            status_data = status_response.json()
+                            data.extend(status_data.get('data', []))
+                        except Exception as e:
+                            logger.error(f"Error during pagination request: {e}")
+                            break
+                        status_data.pop('next', None)
+                    status_data['data'] = data
+
             return {
                 'success': True,
-                'status': data.get('status'),
-                'total': data.get('total'),
-                'completed': data.get('completed'),
-                'creditsUsed': data.get('creditsUsed'),
-                'expiresAt': data.get('expiresAt'),
-                'next': data.get('next'),
-                'data': data.get('data'),
-                'error': data.get('error')
+                'status': status_data.get('status'),
+                'total': status_data.get('total'),
+                'completed': status_data.get('completed'),
+                'creditsUsed': status_data.get('creditsUsed'),
+                'expiresAt': status_data.get('expiresAt'),
+                'data': status_data.get('data'),
+                'error': status_data.get('error'),
+                'next': status_data.get('next', None)
             }
         else:
             self._handle_error(response, 'check batch scrape status')

{firecrawl-1.4.0 → firecrawl-1.5.0}/firecrawl.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 1.4.0
+Version: 1.5.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
@@ -210,11 +210,11 @@ print(batch_scrape_result)
 
 ### Checking batch scrape status
 
-To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
+To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_status` method. It takes the job ID as a parameter and returns the current status of the batch scrape job.
 
 ```python
 id = batch_scrape_result['id']
-status = app.check_batch_scrape_job(id)
+status = app.check_batch_scrape_status(id)
 ```
 
 ### Batch scrape with WebSockets

{firecrawl-1.4.0 → firecrawl-1.5.0}/firecrawl.egg-info/SOURCES.txt

@@ -8,4 +8,8 @@ firecrawl.egg-info/PKG-INFO
 firecrawl.egg-info/SOURCES.txt
 firecrawl.egg-info/dependency_links.txt
 firecrawl.egg-info/requires.txt
-firecrawl.egg-info/top_level.txt
+firecrawl.egg-info/top_level.txt
+firecrawl/__tests__/e2e_withAuth/__init__.py
+firecrawl/__tests__/e2e_withAuth/test.py
+firecrawl/__tests__/v1/e2e_withAuth/__init__.py
+firecrawl/__tests__/v1/e2e_withAuth/test.py

firecrawl-1.5.0/firecrawl.egg-info/top_level.txt

@@ -0,0 +1 @@
+firecrawl

firecrawl-1.4.0/firecrawl/__init__.py

@@ -1,57 +0,0 @@
-"""
-This is the Firecrawl package.
-
-This package provides a Python SDK for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs.
-
-For more information visit https://github.com/firecrawl/
-"""
-
-import logging
-import os
-
-from .firecrawl import FirecrawlApp
-
-__version__ = "1.4.0"
-
-# Define the logger for the Firecrawl project
-logger: logging.Logger = logging.getLogger("firecrawl")
-
-
-def _basic_config() -> None:
-    """Set up basic configuration for logging with a specific format and date format."""
-    try:
-        logging.basicConfig(
-            format="[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-    except Exception as e:
-        logger.error("Failed to configure logging: %s", e)
-
-
-def setup_logging() -> None:
-    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    env = os.environ.get(
-        "FIRECRAWL_LOGGING_LEVEL", "INFO"
-    ).upper()  # Default to 'INFO' level
-    _basic_config()
-
-    if env == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif env == "INFO":
-        logger.setLevel(logging.INFO)
-    elif env == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif env == "ERROR":
-        logger.setLevel(logging.ERROR)
-    elif env == "CRITICAL":
-        logger.setLevel(logging.CRITICAL)
-    else:
-        logger.setLevel(logging.INFO)
-        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-
-# Initialize logging configuration when the module is imported
-setup_logging()
-logger.debug("Debugging logger setup")

firecrawl-1.4.0/firecrawl.egg-info/top_level.txt

@@ -1,3 +0,0 @@
-build
-dist
-firecrawl