firecrawl 0.0.19__py3-none-any.whl → 1.1.1__py3-none-any.whl

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp
 
-__version__ = "0.0.19"
+__version__ = "1.1.1"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/__tests__/e2e_withAuth/test.py

@@ -7,7 +7,7 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-API_URL = "http://127.0.0.1:3002";
+API_URL = "http://127.0.0.1:3002"
 ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
 TEST_API_KEY = os.getenv('TEST_API_KEY')
 
@@ -20,32 +20,34 @@ FirecrawlApp = firecrawl.FirecrawlApp
 
 def test_no_api_key():
     with pytest.raises(Exception) as excinfo:
-      invalid_app = FirecrawlApp(api_url=API_URL)
+      invalid_app = FirecrawlApp(api_url=API_URL, version='v0')
     assert "No API key provided" in str(excinfo.value)
 
 def test_scrape_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
     with pytest.raises(Exception) as excinfo:
         invalid_app.scrape_url('https://firecrawl.dev')
     assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
 
 def test_blocklisted_url():
     blocklisted_url = "https://facebook.com/fake-test"
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     with pytest.raises(Exception) as excinfo:
         app.scrape_url(blocklisted_url)
     assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
 
 def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
+    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')
     response = app.scrape_url('https://roastmywebsite.ai')
     assert response is not None
     assert 'content' in response
     assert "_Roast_" in response['content']
 
 def test_scrape_url_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.scrape_url('https://roastmywebsite.ai')
+    print(response)
+
     assert response is not None
     assert 'content' in response
     assert 'markdown' in response
@@ -54,7 +56,7 @@ def test_scrape_url_e2e():
     assert "_Roast_" in response['content']
 
 def test_successful_response_with_valid_api_key_and_include_html():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.scrape_url('https://roastmywebsite.ai', {'pageOptions': {'includeHtml': True}})
     assert response is not None
     assert 'content' in response
@@ -66,7 +68,7 @@ def test_successful_response_with_valid_api_key_and_include_html():
     assert "<h1" in response['html']
 
 def test_successful_response_for_valid_scrape_with_pdf_file():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
     assert response is not None
     assert 'content' in response
@@ -74,7 +76,7 @@ def test_successful_response_for_valid_scrape_with_pdf_file():
     assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
 
 def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
     time.sleep(6)  # wait for 6 seconds
     assert response is not None
@@ -83,20 +85,20 @@ def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_ext
     assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['content']
 
 def test_crawl_url_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
     with pytest.raises(Exception) as excinfo:
         invalid_app.crawl_url('https://firecrawl.dev')
     assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
 
 def test_should_return_error_for_blocklisted_url():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     blocklisted_url = "https://twitter.com/fake-test"
     with pytest.raises(Exception) as excinfo:
         app.crawl_url(blocklisted_url)
     assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
 
 def test_crawl_url_wait_for_completion_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
     assert response is not None
     assert len(response) > 0
@@ -104,7 +106,7 @@ def test_crawl_url_wait_for_completion_e2e():
     assert "_Roast_" in response[0]['content']
 
 def test_crawl_url_with_idempotency_key_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     uniqueIdempotencyKey = str(uuid4())
     response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
     assert response is not None
@@ -117,7 +119,7 @@ def test_crawl_url_with_idempotency_key_e2e():
     assert "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used" in str(excinfo.value) 
 
 def test_check_crawl_status_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, False)
     assert response is not None
     assert 'jobId' in response
@@ -131,21 +133,21 @@ def test_check_crawl_status_e2e():
     assert len(status_response['data']) > 0
 
 def test_search_e2e():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
     response = app.search("test query")
     assert response is not None
     assert 'content' in response[0]
     assert len(response) > 2
 
 def test_search_invalid_api_key():
-    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key", version='v0')
     with pytest.raises(Exception) as excinfo:
         invalid_app.search("test query")
     assert "Unexpected error during search: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
 
 def test_llm_extraction():
-    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.scrape_url("https://mendable.ai", {
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
+    response = app.scrape_url("https://firecrawl.dev", {
         'extractorOptions': {
             'mode': 'llm-extraction',
             'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",

firecrawl/__tests__/v1/e2e_withAuth/test.py

@@ -0,0 +1,352 @@
+import importlib.util
+import pytest
+import time
+import os
+from uuid import uuid4
+from dotenv import load_dotenv
+from datetime import datetime
+
+load_dotenv()
+
+API_URL = "http://127.0.0.1:3002";
+ABSOLUTE_FIRECRAWL_PATH = "firecrawl/firecrawl.py"
+TEST_API_KEY = os.getenv('TEST_API_KEY')
+
+print(f"ABSOLUTE_FIRECRAWL_PATH: {ABSOLUTE_FIRECRAWL_PATH}")
+
+spec = importlib.util.spec_from_file_location("FirecrawlApp", ABSOLUTE_FIRECRAWL_PATH)
+firecrawl = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(firecrawl)
+FirecrawlApp = firecrawl.FirecrawlApp
+
+def test_no_api_key():
+    with pytest.raises(Exception) as excinfo:
+      invalid_app = FirecrawlApp(api_url=API_URL)
+    assert "No API key provided" in str(excinfo.value)
+
+def test_scrape_url_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.scrape_url('https://firecrawl.dev')
+    assert "Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_blocklisted_url():
+    blocklisted_url = "https://facebook.com/fake-test"
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    with pytest.raises(Exception) as excinfo:
+        app.scrape_url(blocklisted_url)
+    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
+
+def test_successful_response_with_valid_preview_token():
+    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
+    response = app.scrape_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert "_Roast_" in response['markdown']
+    assert "content" not in response
+    assert "html" not in response
+    assert "metadata" in response
+    assert "links" not in response
+    assert "rawHtml" not in response
+
+def test_successful_response_for_valid_scrape():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.scrape_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert 'markdown' in response
+    assert "_Roast_" in response['markdown']
+    assert 'metadata' in response
+    assert 'content' not in response
+    assert 'html' not in response
+    assert 'rawHtml' not in response
+    assert 'screenshot' not in response
+    assert 'links' not in response
+
+def test_successful_response_with_valid_api_key_and_options():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    params = {
+        'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
+        'headers': {'x-key': 'test'},
+        'includeTags': ['h1'],
+        'excludeTags': ['h2'],
+        'onlyMainContent': True,
+        'timeout': 30000,
+        'waitFor': 1000
+    }
+    response = app.scrape_url('https://roastmywebsite.ai', params)
+    assert response is not None
+    assert 'content' not in response
+    assert 'markdown' in response
+    assert 'html' in response
+    assert 'rawHtml' in response
+    assert 'screenshot' in response
+    assert 'links' in response
+    assert "_Roast_" in response['markdown']
+    assert "<h1" in response['html']
+    assert "<h1" in response['rawHtml']
+    assert "https://" in response['screenshot']
+    assert len(response['links']) > 0
+    assert "https://" in response['links'][0]
+    assert 'metadata' in response
+    assert 'title' in response['metadata']
+    assert 'description' in response['metadata']
+    assert 'keywords' in response['metadata']
+    assert 'robots' in response['metadata']
+    assert 'ogTitle' in response['metadata']
+    assert 'ogDescription' in response['metadata']
+    assert 'ogUrl' in response['metadata']
+    assert 'ogImage' in response['metadata']
+    assert 'ogLocaleAlternate' in response['metadata']
+    assert 'ogSiteName' in response['metadata']
+    assert 'sourceURL' in response['metadata']
+    assert 'statusCode' in response['metadata']
+    assert 'pageStatusCode' not in response['metadata']
+    assert 'pageError' not in response['metadata']
+    assert 'error' not in response['metadata']
+    assert response['metadata']['title'] == "Roast My Website"
+    assert response['metadata']['description'] == "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"
+    assert response['metadata']['keywords'] == "Roast My Website,Roast,Website,GitHub,Firecrawl"
+    assert response['metadata']['robots'] == "follow, index"
+    assert response['metadata']['ogTitle'] == "Roast My Website"
+    assert response['metadata']['ogDescription'] == "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"
+    assert response['metadata']['ogUrl'] == "https://www.roastmywebsite.ai"
+    assert response['metadata']['ogImage'] == "https://www.roastmywebsite.ai/og.png"
+    assert response['metadata']['ogLocaleAlternate'] == []
+    assert response['metadata']['ogSiteName'] == "Roast My Website"
+    assert response['metadata']['sourceURL'] == "https://roastmywebsite.ai"
+    assert response['metadata']['statusCode'] == 200
+
+def test_successful_response_for_valid_scrape_with_pdf_file():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf')
+    assert response is not None
+    assert 'content' not in response
+    assert 'metadata' in response
+    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']
+
+def test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001')
+    time.sleep(1)  # wait for 1 second
+    assert response is not None
+    assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']
+
+def test_crawl_url_invalid_api_key():
+    invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.crawl_url('https://firecrawl.dev')
+    assert "Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_should_return_error_for_blocklisted_url():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    blocklisted_url = "https://twitter.com/fake-test"
+    with pytest.raises(Exception) as excinfo:
+        app.crawl_url(blocklisted_url)
+    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
+
+def test_crawl_url_wait_for_completion_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30)
+    assert response is not None
+    assert 'total' in response
+    assert response['total'] > 0
+    assert 'creditsUsed' in response
+    assert response['creditsUsed'] > 0
+    assert 'expiresAt' in response
+    assert datetime.strptime(response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+    assert 'status' in response
+    assert response['status'] == 'completed'
+    assert 'next' not in response
+    assert len(response['data']) > 0
+    assert 'markdown' in response['data'][0]
+    assert "_Roast_" in response['data'][0]['markdown']
+    assert 'content' not in response['data'][0]
+    assert 'html' not in response['data'][0]
+    assert 'rawHtml' not in response['data'][0]
+    assert 'screenshot' not in response['data'][0]
+    assert 'links' not in response['data'][0]
+    assert 'metadata' in response['data'][0]
+    assert 'title' in response['data'][0]['metadata']
+    assert 'description' in response['data'][0]['metadata']
+    assert 'language' in response['data'][0]['metadata']
+    assert 'sourceURL' in response['data'][0]['metadata']
+    assert 'statusCode' in response['data'][0]['metadata']
+    assert 'error' not in response['data'][0]['metadata']
+
+def test_crawl_url_with_options_and_wait_for_completion():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.crawl_url('https://roastmywebsite.ai', {
+        'excludePaths': ['blog/*'],
+        'includePaths': ['/'],
+        'maxDepth': 2,
+        'ignoreSitemap': True,
+        'limit': 10,
+        'allowBackwardLinks': True,
+        'allowExternalLinks': True,
+        'scrapeOptions': {
+            'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
+            'headers': {"x-key": "test"},
+            'includeTags': ['h1'],
+            'excludeTags': ['h2'],
+            'onlyMainContent': True,
+            'waitFor': 1000
+        }
+    }, True, 30)
+    assert response is not None
+    assert 'total' in response
+    assert response['total'] > 0
+    assert 'creditsUsed' in response
+    assert response['creditsUsed'] > 0
+    assert 'expiresAt' in response
+    assert datetime.strptime(response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+    assert 'status' in response
+    assert response['status'] == 'completed'
+    assert 'next' not in response
+    assert len(response['data']) > 0
+    assert 'markdown' in response['data'][0]
+    assert "_Roast_" in response['data'][0]['markdown']
+    assert 'content' not in response['data'][0]
+    assert 'html' in response['data'][0]
+    assert "<h1" in response['data'][0]['html']
+    assert 'rawHtml' in response['data'][0]
+    assert "<h1" in response['data'][0]['rawHtml']
+    assert 'screenshot' in response['data'][0]
+    assert "https://" in response['data'][0]['screenshot']
+    assert 'links' in response['data'][0]
+    assert len(response['data'][0]['links']) > 0
+    assert 'metadata' in response['data'][0]
+    assert 'title' in response['data'][0]['metadata']
+    assert 'description' in response['data'][0]['metadata']
+    assert 'language' in response['data'][0]['metadata']
+    assert 'sourceURL' in response['data'][0]['metadata']
+    assert 'statusCode' in response['data'][0]['metadata']
+    assert 'error' not in response['data'][0]['metadata']
+
+def test_crawl_url_with_idempotency_key_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    uniqueIdempotencyKey = str(uuid4())
+    response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, False, 2, uniqueIdempotencyKey)
+    assert response is not None
+    assert 'id' in response
+
+    with pytest.raises(Exception) as excinfo:
+        app.crawl_url('https://firecrawl.dev', {'excludePaths': ['blog/*']}, True, 2, uniqueIdempotencyKey)
+    assert "Idempotency key already used" in str(excinfo.value)
+
+def test_check_crawl_status_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    response = app.crawl_url('https://firecrawl.dev', {'scrapeOptions': {'formats': ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}}, False)
+    assert response is not None
+    assert 'id' in response
+    
+    max_checks = 15
+    checks = 0
+    status_response = app.check_crawl_status(response['id'])
+    
+    while status_response['status'] == 'scraping' and checks < max_checks:
+        time.sleep(1)  # wait for 1 second
+        assert 'partial_data' not in status_response
+        assert 'current' not in status_response
+        assert 'data' in status_response
+        assert 'total' in status_response
+        assert 'creditsUsed' in status_response
+        assert 'expiresAt' in status_response
+        assert 'status' in status_response
+        assert 'next' in status_response
+        assert status_response['total'] > 0
+        assert status_response['creditsUsed'] > 0
+        assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+        assert status_response['status'] == 'scraping'
+        assert '/v1/crawl/' in status_response['next']
+        status_response = app.check_crawl_status(response['id'])
+        checks += 1
+
+    assert status_response is not None
+    assert 'total' in status_response
+    assert status_response['total'] > 0
+    assert 'creditsUsed' in status_response
+    assert status_response['creditsUsed'] > 0
+    assert 'expiresAt' in status_response
+    assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
+    assert 'status' in status_response
+    assert status_response['status'] == 'completed'
+    assert len(status_response['data']) > 0
+    assert 'markdown' in status_response['data'][0]
+    assert len(status_response['data'][0]['markdown']) > 10
+    assert 'content' not in status_response['data'][0]
+    assert 'html' in status_response['data'][0]
+    assert "<div" in status_response['data'][0]['html']
+    assert 'rawHtml' in status_response['data'][0]
+    assert "<div" in status_response['data'][0]['rawHtml']
+    assert 'screenshot' in status_response['data'][0]
+    assert "https://" in status_response['data'][0]['screenshot']
+    assert 'links' in status_response['data'][0]
+    assert status_response['data'][0]['links'] is not None
+    assert len(status_response['data'][0]['links']) > 0
+    assert 'metadata' in status_response['data'][0]
+    assert 'title' in status_response['data'][0]['metadata']
+    assert 'description' in status_response['data'][0]['metadata']
+    assert 'language' in status_response['data'][0]['metadata']
+    assert 'sourceURL' in status_response['data'][0]['metadata']
+    assert 'statusCode' in status_response['data'][0]['metadata']
+    assert 'error' not in status_response['data'][0]['metadata']
+
+def test_invalid_api_key_on_map():
+    invalid_app = FirecrawlApp(api_key="invalid_api_key", api_url=API_URL)
+    with pytest.raises(Exception) as excinfo:
+        invalid_app.map_url('https://roastmywebsite.ai')
+    assert "Unauthorized: Invalid token" in str(excinfo.value)
+
+def test_blocklisted_url_on_map():
+    app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
+    blocklisted_url = "https://facebook.com/fake-test"
+    with pytest.raises(Exception) as excinfo:
+        app.map_url(blocklisted_url)
+    assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
+
+def test_successful_response_with_valid_preview_token_on_map():
+    app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)
+    response = app.map_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert len(response) > 0
+
+def test_successful_response_for_valid_map():
+    app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
+    response = app.map_url('https://roastmywebsite.ai')
+    assert response is not None
+    assert len(response) > 0
+    assert any("https://" in link for link in response)
+    filtered_links = [link for link in response if "roastmywebsite.ai" in link]
+    assert len(filtered_links) > 0
+
+def test_search_e2e():
+    app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+    with pytest.raises(NotImplementedError) as excinfo:
+        app.search("test query")
+    assert "Search is not supported in v1" in str(excinfo.value)
+
+# def test_llm_extraction():
+#     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+#     response = app.scrape_url("https://mendable.ai", {
+#         'extractorOptions': {
+#             'mode': 'llm-extraction',
+#             'extractionPrompt': "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+#             'extractionSchema': {
+#                 'type': 'object',
+#                 'properties': {
+#                     'company_mission': {'type': 'string'},
+#                     'supports_sso': {'type': 'boolean'},
+#                     'is_open_source': {'type': 'boolean'}
+#                 },
+#                 'required': ['company_mission', 'supports_sso', 'is_open_source']
+#             }
+#         }
+#     })
+#     assert response is not None
+#     assert 'llm_extraction' in response
+#     llm_extraction = response['llm_extraction']
+#     assert 'company_mission' in llm_extraction
+#     assert isinstance(llm_extraction['supports_sso'], bool)
+#     assert isinstance(llm_extraction['is_open_source'], bool)
+
+
+    

firecrawl/firecrawl.py

@@ -12,31 +12,30 @@ Classes:
 import logging
 import os
 import time
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, List
+import asyncio
+import json
 
 import requests
+import websockets
 
 logger : logging.Logger = logging.getLogger("firecrawl")
 
 class FirecrawlApp:
-    """
-    Initialize the FirecrawlApp instance.
-
-    Args:
-        api_key (Optional[str]): API key for authenticating with the Firecrawl API.
-        api_url (Optional[str]): Base URL for the Firecrawl API.
-    """
     def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
-        self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
-        if self.api_key is None:
-            logger.warning("No API key provided")
-            raise ValueError('No API key provided')
-        else:
-            logger.debug("Initialized FirecrawlApp with API key: %s", self.api_key)
-
-        self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')
-        if self.api_url != 'https://api.firecrawl.dev':
-            logger.debug("Initialized FirecrawlApp with API URL: %s", self.api_url)
+      """
+      Initialize the FirecrawlApp instance with API key, API URL.
+
+      Args:
+          api_key (Optional[str]): API key for authenticating with the Firecrawl API.
+          api_url (Optional[str]): Base URL for the Firecrawl API.
+      """
+      self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
+      self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')
+      if self.api_key is None:
+          logger.warning("No API key provided")
+          raise ValueError('No API key provided')
+      logger.debug(f"Initialized FirecrawlApp with API key: {self.api_key}")
 
     def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any:
         """
@@ -75,9 +74,11 @@ class FirecrawlApp:
             for key, value in params.items():
                 if key != 'extractorOptions':
                     scrape_params[key] = value
+
+        endpoint = f'/v1/scrape'
         # Make the POST request with the prepared headers and JSON data
         response = requests.post(
-            f'{self.api_url}/v0/scrape',
+            f'{self.api_url}{endpoint}',
             headers=headers,
             json=scrape_params,
         )
@@ -102,32 +103,14 @@ class FirecrawlApp:
             Any: The search results if the request is successful.
 
         Raises:
+            NotImplementedError: If the search request is attempted on API version v1.
             Exception: If the search request fails.
         """
-        headers = self._prepare_headers()
-        json_data = {'query': query}
-        if params:
-            json_data.update(params)
-        response = requests.post(
-            f'{self.api_url}/v0/search',
-            headers=headers,
-            json=json_data
-        )
-        if response.status_code == 200:
-            response = response.json()
-
-            if response['success'] and 'data' in response:
-                return response['data']
-            else:
-                raise Exception(f'Failed to search. Error: {response["error"]}')
-
-        else:
-            self._handle_error(response, 'search')
+        raise NotImplementedError("Search is not supported in v1.")
 
     def crawl_url(self, url: str,
                   params: Optional[Dict[str, Any]] = None,
-                  wait_until_done: bool = True,
-                  poll_interval: int = 2,
+                  poll_interval: Optional[int] = 2,
                   idempotency_key: Optional[str] = None) -> Any:
         """
         Initiate a crawl job for the specified URL using the Firecrawl API.
@@ -135,8 +118,7 @@ class FirecrawlApp:
         Args:
             url (str): The URL to crawl.
             params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
-            wait_until_done (bool): Whether to wait until the crawl job is completed.
-            poll_interval (int): Time in seconds between status checks when waiting for job completion.
+            poll_interval (Optional[int]): Time in seconds between status checks when waiting for job completion. Defaults to 2 seconds.
             idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
 
         Returns:
@@ -145,26 +127,49 @@ class FirecrawlApp:
         Raises:
             Exception: If the crawl job initiation or monitoring fails.
         """
+        endpoint = f'/v1/crawl'
         headers = self._prepare_headers(idempotency_key)
         json_data = {'url': url}
         if params:
             json_data.update(params)
-        response = self._post_request(f'{self.api_url}/v0/crawl', json_data, headers)
+        response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)
         if response.status_code == 200:
-            job_id = response.json().get('jobId')
-            if wait_until_done:
-                return self._monitor_job_status(job_id, headers, poll_interval)
-            else:
-                return {'jobId': job_id}
+            id = response.json().get('id')
+            return self._monitor_job_status(id, headers, poll_interval)
+
         else:
             self._handle_error(response, 'start crawl job')
 
-    def check_crawl_status(self, job_id: str) -> Any:
+
+    def async_crawl_url(self, url: str, params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Initiate a crawl job asynchronously.
+
+        Args:
+            url (str): The URL to crawl.
+            params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
+            idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
+
+        Returns:
+            Dict[str, Any]: The response from the crawl initiation request.
+        """
+        endpoint = f'/v1/crawl'
+        headers = self._prepare_headers(idempotency_key)
+        json_data = {'url': url}
+        if params:
+            json_data.update(params)
+        response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)
+        if response.status_code == 200:
+            return response.json()
+        else:
+            self._handle_error(response, 'start crawl job')
+
+    def check_crawl_status(self, id: str) -> Any:
         """
         Check the status of a crawl job using the Firecrawl API.
 
         Args:
-            job_id (str): The ID of the crawl job.
+            id (str): The ID of the crawl job.
 
         Returns:
             Any: The status of the crawl job.
@@ -172,13 +177,79 @@ class FirecrawlApp:
         Raises:
             Exception: If the status check request fails.
         """
+        endpoint = f'/v1/crawl/{id}'
+
         headers = self._prepare_headers()
-        response = self._get_request(f'{self.api_url}/v0/crawl/status/{job_id}', headers)
+        response = self._get_request(f'{self.api_url}{endpoint}', headers)
         if response.status_code == 200:
-            return response.json()
+            data = response.json()
+            return {
+                'success': True,
+                'status': data.get('status'),
+                'total': data.get('total'),
+                'completed': data.get('completed'),
+                'creditsUsed': data.get('creditsUsed'),
+                'expiresAt': data.get('expiresAt'),
+                'next': data.get('next'),
+                'data': data.get('data'),
+                'error': data.get('error')
+            }
         else:
             self._handle_error(response, 'check crawl status')
 
+    def crawl_url_and_watch(self, url: str, params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> 'CrawlWatcher':
+        """
+        Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.
+
+        Args:
+            url (str): The URL to crawl.
+            params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
+            idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
+
+        Returns:
+            CrawlWatcher: An instance of CrawlWatcher to monitor the crawl job.
+        """
+        crawl_response = self.async_crawl_url(url, params, idempotency_key)
+        if crawl_response['success'] and 'id' in crawl_response:
+            return CrawlWatcher(crawl_response['id'], self)
+        else:
+            raise Exception("Crawl job failed to start")
+
+    def map_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        """
+        Perform a map search using the Firecrawl API.
+
+        Args:
+            url (str): The URL to perform the map search on.
+            params (Optional[Dict[str, Any]]): Additional parameters for the map search.
+
+        Returns:
+            Any: The result of the map search, typically a dictionary containing mapping data.
+        """
+        endpoint = f'/v1/map'
+        headers = self._prepare_headers()
+
+        # Prepare the base scrape parameters with the URL
+        json_data = {'url': url}
+        if params:
+            json_data.update(params)
+        
+        # Make the POST request with the prepared headers and JSON data
+        response = requests.post(
+            f'{self.api_url}{endpoint}',
+            headers=headers,
+            json=json_data,
+        )
+        if response.status_code == 200:
+            response = response.json()
+            print(response)
+            if response['success'] and 'links' in response:
+                return response['links']
+            else:
+                raise Exception(f'Failed to map URL. Error: {response["error"]}')
+        else:
+            self._handle_error(response, 'map')
+
     def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
         """
         Prepare the headers for API requests.
@@ -257,15 +328,14 @@ class FirecrawlApp:
                 return response
         return response
 
-    def _monitor_job_status(self, job_id: str, headers: Dict[str, str], poll_interval: int) -> Any:
+    def _monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int) -> Any:
         """
         Monitor the status of a crawl job until completion.
 
         Args:
-            job_id (str): The ID of the crawl job.
+            id (str): The ID of the crawl job.
             headers (Dict[str, str]): The headers to include in the status check requests.
             poll_interval (int): Secounds between status checks.
-
         Returns:
             Any: The crawl results if the job is completed successfully.
 
@@ -273,15 +343,17 @@ class FirecrawlApp:
             Exception: If the job fails or an error occurs during status checks.
         """
         while True:
-            status_response = self._get_request(f'{self.api_url}/v0/crawl/status/{job_id}', headers)
+            api_url = f'{self.api_url}/v1/crawl/{id}'
+
+            status_response = self._get_request(api_url, headers)
             if status_response.status_code == 200:
                 status_data = status_response.json()
                 if status_data['status'] == 'completed':
                     if 'data' in status_data:
-                        return status_data['data']
+                        return status_data
                     else:
                         raise Exception('Crawl job completed but no data was returned')
-                elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting']:
+                elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:
                     poll_interval=max(poll_interval,2)
                     time.sleep(poll_interval)  # Wait for the specified interval before checking again
                 else:
@@ -300,19 +372,66 @@ class FirecrawlApp:
         Raises:
             Exception: An exception with a message containing the status code and error details from the response.
         """
-        error_message = response.json().get('error', 'No additional error details provided.')
+        error_message = response.json().get('error', 'No error message provided.')
+        error_details = response.json().get('details', 'No additional error details provided.')
 
         if response.status_code == 402:
-            message = f"Payment Required: Failed to {action}. {error_message}"
+            message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
         elif response.status_code == 408:
-            message = f"Request Timeout: Failed to {action} as the request timed out. {error_message}"
+            message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
         elif response.status_code == 409:
-            message = f"Conflict: Failed to {action} due to a conflict. {error_message}"
+            message = f"Conflict: Failed to {action} due to a conflict. {error_message} - {error_details}"
         elif response.status_code == 500:
-            message = f"Internal Server Error: Failed to {action}. {error_message}"
+            message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
         else:
-            message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message}"
+            message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message} - {error_details}"
 
         # Raise an HTTPError with the custom message and attach the response
         raise requests.exceptions.HTTPError(message, response=response)
-    
+
+class CrawlWatcher:
+    def __init__(self, id: str, app: FirecrawlApp):
+        self.id = id
+        self.app = app
+        self.data: List[Dict[str, Any]] = []
+        self.status = "scraping"
+        self.ws_url = f"{app.api_url.replace('http', 'ws')}/v1/crawl/{id}"
+        self.event_handlers = {
+            'done': [],
+            'error': [],
+            'document': []
+        }
+
+    async def connect(self):
+        async with websockets.connect(self.ws_url, extra_headers={"Authorization": f"Bearer {self.app.api_key}"}) as websocket:
+            await self._listen(websocket)
+
+    async def _listen(self, websocket):
+        async for message in websocket:
+            msg = json.loads(message)
+            await self._handle_message(msg)
+
+    def add_event_listener(self, event_type: str, handler):
+        if event_type in self.event_handlers:
+            self.event_handlers[event_type].append(handler)
+
+    def dispatch_event(self, event_type: str, detail: Dict[str, Any]):
+        if event_type in self.event_handlers:
+            for handler in self.event_handlers[event_type]:
+                handler(detail)
+
+    async def _handle_message(self, msg: Dict[str, Any]):
+        if msg['type'] == 'done':
+            self.status = 'completed'
+            self.dispatch_event('done', {'status': self.status, 'data': self.data})
+        elif msg['type'] == 'error':
+            self.status = 'failed'
+            self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error']})
+        elif msg['type'] == 'catchup':
+            self.status = msg['data']['status']
+            self.data.extend(msg['data'].get('data', []))
+            for doc in self.data:
+                self.dispatch_event('document', doc)
+        elif msg['type'] == 'document':
+            self.data.append(msg['data'])
+            self.dispatch_event('document', msg['data'])

firecrawl-1.1.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Sideguide Technologies Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{firecrawl-0.0.19.dist-info → firecrawl-1.1.1.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 0.0.19
+Version: 1.1.1
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
 Author-email: "Mendable.ai" <nick@mendable.ai>
 Maintainer-email: "Mendable.ai" <nick@mendable.ai>
-License: GNU General Public License v3 (GPLv3)
+License: GNU Affero General Public License v3 (AGPLv3)
 Project-URL: Documentation, https://docs.firecrawl.dev
 Project-URL: Source, https://github.com/mendableai/firecrawl
 Project-URL: Tracker, https://github.com/mendableai/firecrawl/issues
@@ -30,9 +30,14 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Text Processing :: Indexing
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: requests
+Requires-Dist: python-dotenv
+Requires-Dist: websockets
+Requires-Dist: asyncio
+Requires-Dist: nest-asyncio
 
 # Firecrawl Python SDK
 
@@ -51,27 +56,31 @@ pip install firecrawl-py
 1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
 2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.
 
-
 Here's an example of how to use the SDK:
 
 ```python
-from firecrawl import FirecrawlApp
-
-# Initialize the FirecrawlApp with your API key
-app = FirecrawlApp(api_key='your_api_key')
-
-# Scrape a single URL
-url = 'https://mendable.ai'
-scraped_data = app.scrape_url(url)
-
-# Crawl a website
-crawl_url = 'https://mendable.ai'
-params = {
-    'pageOptions': {
-        'onlyMainContent': True
-    }
-}
-crawl_result = app.crawl_url(crawl_url, params=params)
+from firecrawl.firecrawl import FirecrawlApp
+
+app = FirecrawlApp(api_key="fc-YOUR_API_KEY")
+
+# Scrape a website:
+scrape_status = app.scrape_url(
+  'https://firecrawl.dev', 
+  params={'formats': ['markdown', 'html']}
+)
+print(scrape_status)
+
+# Crawl a website:
+crawl_status = app.crawl_url(
+  'https://firecrawl.dev', 
+  params={
+    'limit': 100, 
+    'scrapeOptions': {'formats': ['markdown', 'html']}
+  }, 
+  wait_until_done=True, 
+  poll_interval=30
+)
+print(crawl_status)
 ```
 
 ### Scraping a URL
@@ -82,6 +91,7 @@ To scrape a single URL, use the `scrape_url` method. It takes the URL as a param
 url = 'https://example.com'
 scraped_data = app.scrape_url(url)
 ```
+
 ### Extracting structured data from a URL
 
 With LLM extraction, you can easily extract structured data from any URL. We support pydantic schemas to make it easier for you too. Here is how you to use it:
@@ -89,7 +99,7 @@ With LLM extraction, you can easily extract structured data from any URL. We sup
 ```python
 class ArticleSchema(BaseModel):
     title: str
-    points: int 
+    points: int
     by: str
     commentsURL: str
 
@@ -108,45 +118,77 @@ data = app.scrape_url('https://news.ycombinator.com', {
 print(data["llm_extraction"])
 ```
 
-### Search for a query
+### Crawling a Website
 
-Used to search the web, get the most relevant results, scrap each page and return the markdown.
+To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
 
 ```python
-query = 'what is mendable?'
-search_result = app.search(query)
+idempotency_key = str(uuid.uuid4()) # optional idempotency key
+crawl_result = app.crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, 2, idempotency_key)
+print(crawl_result)
 ```
 
-### Crawling a Website
-
-To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+### Asynchronous Crawl a Website
 
-The `wait_until_done` parameter determines whether the method should wait for the crawl job to complete before returning the result. If set to `True`, the method will periodically check the status of the crawl job until it is completed or the specified `timeout` (in seconds) is reached. If set to `False`, the method will return immediately with the job ID, and you can manually check the status of the crawl job using the `check_crawl_status` method.
+To crawl a website asynchronously, use the `async_crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
 
 ```python
-crawl_url = 'https://example.com'
-params = {
-    'crawlerOptions': {
-        'excludes': ['blog/*'],
-        'includes': [], # leave empty for all pages
-        'limit': 1000,
-    },
-    'pageOptions': {
-        'onlyMainContent': True
-    }
-}
-crawl_result = app.crawl_url(crawl_url, params=params, wait_until_done=True, timeout=5)
+crawl_result = app.async_crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, "")
+print(crawl_result)
 ```
 
-If `wait_until_done` is set to `True`, the `crawl_url` method will return the crawl result once the job is completed. If the job fails or is stopped, an exception will be raised.
-
 ### Checking Crawl Status
 
 To check the status of a crawl job, use the `check_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
 
 ```python
-job_id = crawl_result['jobId']
-status = app.check_crawl_status(job_id)
+id = crawl_result['id']
+status = app.check_crawl_status(id)
+```
+
+### Map a Website
+
+Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap.
+
+```python
+# Map a website:
+map_result = app.map_url('https://example.com')
+print(map_result)
+```
+
+### Crawl a website with WebSockets
+
+To crawl a website with WebSockets, use the `crawl_url_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+
+```python
+# inside an async function...
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+    print("DOC", detail)
+
+def on_error(detail):
+    print("ERR", detail['error'])
+
+def on_done(detail):
+    print("DONE", detail['status'])
+
+    # Function to start the crawl and watch process
+async def start_crawl_and_watch():
+    # Initiate the crawl job and get the watcher
+    watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
+
+    # Add event listeners
+    watcher.add_event_listener("document", on_document)
+    watcher.add_event_listener("error", on_error)
+    watcher.add_event_listener("done", on_done)
+
+    # Start the watcher
+    await watcher.connect()
+
+# Run the event loop
+await start_crawl_and_watch()
 ```
 
 ## Error Handling
@@ -162,20 +204,27 @@ To ensure the functionality of the Firecrawl Python SDK, we have included end-to
 To run the tests, execute the following commands:
 
 Install pytest:
+
 ```bash
 pip install pytest
 ```
 
 Run:
+
 ```bash
 pytest firecrawl/__tests__/e2e_withAuth/test.py
 ```
 
-
 ## Contributing
 
 Contributions to the Firecrawl Python SDK are welcome! If you find any issues or have suggestions for improvements, please open an issue or submit a pull request on the GitHub repository.
 
 ## License
 
-The Firecrawl Python SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).
+The Firecrawl Python SDK is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the SDK, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Please note that while this SDK is MIT licensed, it is part of a larger project which may be under different licensing terms. Always refer to the license information in the root directory of the main project for overall licensing details.

firecrawl-1.1.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+firecrawl/__init__.py,sha256=j1M2OH1eJLLBEooOLOVrfgqyDyuOqSHdSRbxKE6FH2E,1682
+firecrawl/firecrawl.py,sha256=XIO5Pyw2w9i32f-fIjO5hgqLvPV7ZSi66pR0JQjFzVI,17713
+firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/e2e_withAuth/test.py,sha256=L-umFR3WyrJso1EwqkxjbTMr5AEI4t5zDfhQcCzitOI,7911
+firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=KQMmGAtJAIafja6EGtJ-W9162w2Hm6PNjqKl3_RQXLA,16456
+firecrawl-1.1.1.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl-1.1.1.dist-info/METADATA,sha256=I7nFWbZ4zD9GRoeQDtGfYVEnGEKNPT7ltkA0D4gUH0g,8297
+firecrawl-1.1.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl-1.1.1.dist-info/top_level.txt,sha256=jTvz79zWhiyAezfmmHe4FQ-hR60C59UU5FrjMjijLu8,10
+firecrawl-1.1.1.dist-info/RECORD,,

{firecrawl-0.0.19.dist-info → firecrawl-1.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.2.0)
+Generator: bdist_wheel (0.38.4)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

firecrawl-0.0.19.dist-info/RECORD

@@ -1,8 +0,0 @@
-firecrawl/__init__.py,sha256=ixdimo55Kkf1ccxo31pemLUm16hBOG5GvgEen9S3qNc,1683
-firecrawl/firecrawl.py,sha256=wfy2KXSrLoGlj8HKEP3VKG_u38kRbUbcmRQsMgza5I0,12767
-firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-firecrawl/__tests__/e2e_withAuth/test.py,sha256=vZlhdURkMXY1-2_FBrKwT6D4WbH7HEf_xMEd0X63krQ,7665
-firecrawl-0.0.19.dist-info/METADATA,sha256=M92W_xzUtd6k5-789ti_yyfC88ZXZAdXq7Beqqx6iqc,6264
-firecrawl-0.0.19.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-firecrawl-0.0.19.dist-info/top_level.txt,sha256=jTvz79zWhiyAezfmmHe4FQ-hR60C59UU5FrjMjijLu8,10
-firecrawl-0.0.19.dist-info/RECORD,,