firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/__init__.py

@@ -1,19 +1,23 @@
 """
-This is the Firecrawl package.
+Firecrawl Python SDK
 
-This package provides a Python SDK for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs.
-
-For more information visit https://github.com/firecrawl/
 """
 
 import logging
 import os
 
-from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
+from .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlApp
+from .v2.watcher import Watcher
+from .v2.watcher_async import AsyncWatcher
+from .v1 import (
+    V1FirecrawlApp,
+    AsyncV1FirecrawlApp,
+    V1JsonConfig,
+    V1ScrapeOptions,
+    V1ChangeTrackingOptions,
+)
 
-__version__ = "2.16.5"
+__version__ = "3.0.3"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")
@@ -27,17 +31,14 @@ def _configure_logger() -> None:
     format to the firecrawl logger.
     """
     try:
-        # Create the formatter
         formatter = logging.Formatter(
             "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
             datefmt="%Y-%m-%d %H:%M:%S",
         )
 
-        # Create the console handler and set the formatter
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(formatter)
 
-        # Add the console handler to the firecrawl logger
         logger.addHandler(console_handler)
     except Exception as e:
         logger.error("Failed to configure logging: %s", e)
@@ -45,20 +46,15 @@ def _configure_logger() -> None:
 
 def setup_logging() -> None:
     """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    # Check if the firecrawl logger already has a handler
     if logger.hasHandlers():
-        return # To prevent duplicate logging
+        return
 
-    # Check if the FIRECRAWL_LOGGING_LEVEL environment variable is set
     if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
-        # Attach a no-op handler to prevent warnings about no handlers
         logger.addHandler(logging.NullHandler()) 
         return
 
-    # Attach the console handler to the firecrawl logger
     _configure_logger()
 
-    # Set the logging level based on the FIRECRAWL_LOGGING_LEVEL environment variable
     if env == "DEBUG":
         logger.setLevel(logging.DEBUG)
     elif env == "INFO":
@@ -73,7 +69,19 @@ def setup_logging() -> None:
         logger.setLevel(logging.INFO)
         logger.warning("Unknown logging level: %s, defaulting to INFO", env)
 
-
-# Initialize logging configuration when the module is imported
 setup_logging()
 logger.debug("Debugging logger setup")
+
+__all__ = [
+    'Firecrawl',
+    'AsyncFirecrawl',
+    'FirecrawlApp',
+    'AsyncFirecrawlApp',
+    'Watcher',
+    'AsyncWatcher',
+    'V1FirecrawlApp',
+    'AsyncV1FirecrawlApp',
+    'V1JsonConfig',
+    'V1ScrapeOptions',
+    'V1ChangeTrackingOptions',
+]

firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py

@@ -0,0 +1,79 @@
+import os
+import asyncio
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_start_and_status():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    job_id = start.id
+
+    deadline = asyncio.get_event_loop().time() + 240
+    status = await client.get_batch_scrape_status(job_id)
+    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_batch_scrape_status(job_id)
+
+    assert status.status in ("completed", "failed", "cancelled")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_wait_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    job = await client.batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_wait_with_all_params():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    job = await client.batch_scrape(
+        [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ],
+        formats=[
+            "markdown",
+            {"type": "json", "prompt": "Extract page title", "schema": json_schema},
+            {"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
+        ],
+        only_main_content=True,
+        mobile=False,
+        ignore_invalid_urls=True,
+        max_concurrency=2,
+        zero_data_retention=False,
+        poll_interval=1,
+        timeout=180,
+    )
+    assert job.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_cancel_batch():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    ok = await client.cancel_batch_scrape(start.id)
+    assert ok is True
+

firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py

@@ -0,0 +1,189 @@
+import os
+import asyncio
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.types import ScrapeOptions
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_start_and_status():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
+    job_id = start.id
+
+    deadline = asyncio.get_event_loop().time() + 180
+    status = await client.get_crawl_status(job_id)
+    while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_crawl_status(job_id)
+
+    assert status.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_all_params():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    # rich scrape options including json format
+    json_schema = {
+        "type": "object",
+        "properties": {"title": {"type": "string"}},
+        "required": ["title"],
+    }
+    status = await client.crawl(
+        url="https://docs.firecrawl.dev",
+        prompt="Extract docs and blog",
+        include_paths=["/docs/*", "/blog/*"],
+        exclude_paths=["/admin/*"],
+        max_discovery_depth=2,
+        ignore_sitemap=False,
+        ignore_query_parameters=True,
+        limit=5,
+        crawl_entire_domain=False,
+        allow_external_links=True,
+        allow_subdomains=True,
+        delay=1,
+        max_concurrency=2,
+        webhook="https://example.com/hook",
+        scrape_options=ScrapeOptions(
+            formats=[
+                "markdown",
+                "rawHtml",
+                {"type": "json", "prompt": "Extract title", "schema": json_schema},
+            ],
+            only_main_content=True,
+            mobile=False,
+            timeout=20000,
+            wait_for=500,
+            skip_tls_verification=False,
+            remove_base64_images=False,
+        ),
+        zero_data_retention=False,
+        poll_interval=2,
+        timeout=180,
+    )
+    assert status.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_start_crawl_with_options():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=5, max_discovery_depth=2)
+    assert start.id is not None and start.url is not None
+
+
+@pytest.mark.asyncio
+async def test_async_start_crawl_with_prompt():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://firecrawl.dev", prompt="Extract all blog posts", limit=3)
+    assert start.id is not None and start.url is not None
+
+
+@pytest.mark.asyncio
+async def test_async_get_crawl_status_shape():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
+    status = await client.get_crawl_status(start.id)
+    assert status.status in ("scraping", "completed", "failed")
+    assert status.completed >= 0
+    assert status.expires_at is not None
+    assert status.next is not None
+    assert isinstance(status.data, list)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_wait():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    job = await client.crawl(url="https://docs.firecrawl.dev", limit=3, max_discovery_depth=2, poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_prompt_and_wait():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    job = await client.crawl(url="https://docs.firecrawl.dev", prompt="Extract all blog posts", limit=3, poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_scrape_options():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    scrape_opts = ScrapeOptions(formats=["markdown", "links"], only_main_content=False, mobile=True)
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
+    assert start.id is not None
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_json_format_object():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    scrape_opts = ScrapeOptions(formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}])
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
+    assert start.id is not None
+
+
+@pytest.mark.asyncio
+async def test_async_cancel_crawl():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
+    cancelled = await client.cancel_crawl(start.id)
+    assert cancelled is True
+
+
+@pytest.mark.asyncio
+async def test_async_get_crawl_errors_and_invalid_job():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
+    errs = await client.get_crawl_errors(start.id)
+    assert hasattr(errs, "errors") and hasattr(errs, "robots_blocked")
+    with pytest.raises(Exception):
+        await client.get_crawl_errors("invalid-job-id-12345")
+
+
+@pytest.mark.asyncio
+async def test_async_active_crawls():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    resp = await client.active_crawls()
+    assert hasattr(resp, "success") and hasattr(resp, "crawls")
+
+
+@pytest.mark.asyncio
+async def test_async_active_crawls_with_running_crawl():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
+    # fetch active crawls and assert our ID is listed
+    active = await client.active_crawls()
+    ids = [c.id for c in active.crawls]
+    assert start.id in ids
+    # cleanup
+    await client.cancel_crawl(start.id)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_params_preview():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    params = await client.crawl_params_preview(
+        url="https://docs.firecrawl.dev",
+        prompt="Extract all blog posts and documentation",
+    )
+    assert params is not None
+    # basic sanity: at least one field should be suggested
+    has_any = any([
+        getattr(params, "limit", None) is not None,
+        getattr(params, "include_paths", None) is not None,
+        getattr(params, "max_discovery_depth", None) is not None,
+    ])
+    assert has_any
+
+

firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py

@@ -0,0 +1,38 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_extract_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    res = await client.extract(urls=["https://docs.firecrawl.dev"], prompt="Extract title")
+    assert res is not None
+
+
+@pytest.mark.asyncio
+async def test_async_extract_with_schema_and_options():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    res = await client.extract(
+        urls=["https://docs.firecrawl.dev"],
+        prompt="Extract title",
+        schema=schema,
+        system_prompt="You are a helpful extractor",
+        allow_external_links=False,
+        enable_web_search=False,
+        show_sources=False,
+        # agent={"model": "FIRE-1", "prompt": "Extract title"}, # Skipping agent test in CI
+    )
+    assert res is not None
+

firecrawl/__tests__/e2e/v2/aio/test_aio_map.py

@@ -0,0 +1,40 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_map_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    resp = await client.map("https://docs.firecrawl.dev")
+    assert hasattr(resp, "links") and isinstance(resp.links, list)
+    if resp.links:
+        first = resp.links[0]
+        assert hasattr(first, "url") and isinstance(first.url, str) and first.url.startswith("http")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sitemap", ["only", "include", "skip"])
+async def test_async_map_with_all_params(sitemap):
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    resp = await client.map(
+        "https://docs.firecrawl.dev",
+        search="docs",
+        include_subdomains=True,
+        limit=10,
+        sitemap=sitemap,
+        timeout=15000,
+    )
+    assert hasattr(resp, "links") and isinstance(resp.links, list)
+    assert len(resp.links) <= 10
+

firecrawl/__tests__/e2e/v2/aio/test_aio_scrape.py

@@ -0,0 +1,137 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.types import Document
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape("https://docs.firecrawl.dev")
+    assert isinstance(doc, Document)
+    assert (
+        (doc.markdown and len(doc.markdown) > 0)
+        or (doc.html and len(doc.html) > 0)
+        or (doc.raw_html and len(doc.raw_html) > 0)
+        or (doc.links is not None)
+        or (doc.screenshot is not None)
+        or (doc.json is not None)
+        or (doc.summary is not None)
+    )
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_all_params():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    # Include multiple formats with configuration
+    json_schema = {
+        "type": "object",
+        "properties": {"title": {"type": "string"}},
+        "required": ["title"],
+    }
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=[
+            "markdown",
+            "rawHtml",
+            {"type": "screenshot", "full_page": False, "quality": 70},
+            {"type": "json", "prompt": "Extract title", "schema": json_schema},
+        ],
+        headers={"User-Agent": "E2E-AIO"},
+        include_tags=["main"],
+        exclude_tags=["nav"],
+        only_main_content=True,
+        timeout=20000,
+        wait_for=500,
+        mobile=False,
+        parsers=["pdf"],
+        actions=[],
+        skip_tls_verification=False,
+        remove_base64_images=False,
+        fast_mode=False,
+        use_mock=None,
+        block_ads=False,
+        proxy="basic",
+        max_age=0,
+        store_in_cache=False,
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_options_markdown():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=["markdown"],
+        only_main_content=False,
+        mobile=False,
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_screenshot_action_viewport():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=[{"type": "screenshot", "full_page": False, "quality": 80, "viewport": {"width": 800, "height": 600}}],
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fmt,expect_field", [
+    ("markdown", "markdown"),
+    ("html", "html"),
+    ("raw_html", "raw_html"),
+    ("links", "links"),
+    ("screenshot", "screenshot"),
+    ("summary", "summary"),
+])
+async def test_async_scrape_basic_formats(fmt, expect_field):
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape("https://docs.firecrawl.dev", formats=[fmt])
+    assert isinstance(doc, Document)
+    if expect_field == "markdown":
+        assert doc.markdown is not None
+    elif expect_field == "html":
+        assert doc.html is not None
+    elif expect_field == "raw_html":
+        assert doc.raw_html is not None
+    elif expect_field == "links":
+        assert isinstance(doc.links, list)
+    elif expect_field == "screenshot":
+        assert doc.screenshot is not None
+    elif expect_field == "summary":
+        assert doc.summary is not None
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_json_format_object():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],
+        only_main_content=True,
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_invalid_url():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    with pytest.raises(ValueError):
+        await client.scrape("")
+