firecrawl 4.12.0__py3-none-any.whl

firecrawl/__init__.py

@@ -0,0 +1,87 @@
+"""
+Firecrawl Python SDK
+
+"""
+
+import logging
+import os
+
+from .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlApp
+from .v2.watcher import Watcher
+from .v2.watcher_async import AsyncWatcher
+from .v1 import (
+    V1FirecrawlApp,
+    AsyncV1FirecrawlApp,
+    V1JsonConfig,
+    V1ScrapeOptions,
+    V1ChangeTrackingOptions,
+)
+
+__version__ = "4.12.0"
+
+# Define the logger for the Firecrawl project
+logger: logging.Logger = logging.getLogger("firecrawl")
+
+
+def _configure_logger() -> None:
+    """
+    Configure the firecrawl logger for console output.
+
+    The function attaches a handler for console output with a specific format and date
+    format to the firecrawl logger.
+    """
+    try:
+        formatter = logging.Formatter(
+            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+
+        logger.addHandler(console_handler)
+    except Exception as e:
+        logger.error("Failed to configure logging: %s", e)
+
+
+def setup_logging() -> None:
+    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
+    if logger.hasHandlers():
+        return
+
+    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
+        logger.addHandler(logging.NullHandler()) 
+        return
+
+    _configure_logger()
+
+    if env == "DEBUG":
+        logger.setLevel(logging.DEBUG)
+    elif env == "INFO":
+        logger.setLevel(logging.INFO)
+    elif env == "WARNING":
+        logger.setLevel(logging.WARNING)
+    elif env == "ERROR":
+        logger.setLevel(logging.ERROR)
+    elif env == "CRITICAL":
+        logger.setLevel(logging.CRITICAL)
+    else:
+        logger.setLevel(logging.INFO)
+        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
+
+setup_logging()
+logger.debug("Debugging logger setup")
+
+__all__ = [
+    'Firecrawl',
+    'AsyncFirecrawl',
+    'FirecrawlApp',
+    'AsyncFirecrawlApp',
+    'Watcher',
+    'AsyncWatcher',
+    'V1FirecrawlApp',
+    'AsyncV1FirecrawlApp',
+    'V1JsonConfig',
+    'V1ScrapeOptions',
+    'V1ChangeTrackingOptions',
+]

firecrawl/__tests__/e2e/v2/aio/conftest.py

@@ -0,0 +1,62 @@
+"""
+Pytest configuration for async E2E tests.
+Ensures environment variables are loaded before any test runs.
+"""
+import os
+import pytest
+from dotenv import load_dotenv, find_dotenv
+
+# Load environment IMMEDIATELY at module import time (before pytest collects tests)
+# This ensures env vars are loaded before the first test runs
+_env_loaded = False
+
+def _ensure_env_loaded():
+    """Ensure environment is loaded exactly once, synchronously."""
+    global _env_loaded
+    if not _env_loaded:
+        # Find and load .env from multiple possible locations
+        env_file = find_dotenv(usecwd=True)
+        if env_file:
+            load_dotenv(env_file, override=True)
+        else:
+            # Try loading from current directory
+            load_dotenv(override=True)
+        _env_loaded = True
+
+# Load env immediately when this module is imported
+_ensure_env_loaded()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def load_environment():
+    """Ensure environment variables are loaded before running any tests."""
+    # Double-check env is loaded (should already be done at import time)
+    _ensure_env_loaded()
+    
+    # Validate required environment variables
+    required_vars = ["API_KEY", "API_URL"]
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    
+    if missing_vars:
+        pytest.skip(f"Skipping E2E tests: Missing required environment variables: {', '.join(missing_vars)}")
+    
+    yield
+
+
+@pytest.fixture(scope="function")
+def api_key():
+    """Provide API key for tests."""
+    key = os.getenv("API_KEY")
+    if not key:
+        pytest.skip("API_KEY not set")
+    return key
+
+
+@pytest.fixture(scope="function")
+def api_url():
+    """Provide API URL for tests."""
+    url = os.getenv("API_URL")
+    if not url:
+        pytest.skip("API_URL not set")
+    return url
+

firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py

@@ -0,0 +1,69 @@
+import asyncio
+import pytest
+from firecrawl import AsyncFirecrawl
+
+
+@pytest.mark.asyncio
+async def test_async_batch_start_and_status(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    job_id = start.id
+
+    deadline = asyncio.get_event_loop().time() + 240
+    status = await client.get_batch_scrape_status(job_id)
+    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_batch_scrape_status(job_id)
+
+    assert status.status in ("completed", "failed", "cancelled")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_wait_minimal(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    job = await client.batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_wait_with_all_params(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    job = await client.batch_scrape(
+        [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ],
+        formats=[
+            "markdown",
+            {"type": "json", "prompt": "Extract page title", "schema": json_schema},
+            {"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
+        ],
+        only_main_content=True,
+        mobile=False,
+        ignore_invalid_urls=True,
+        max_concurrency=2,
+        zero_data_retention=False,
+        poll_interval=1,
+        timeout=180,
+        integration="_e2e-test",
+    )
+    assert job.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_cancel_batch(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    ok = await client.cancel_batch_scrape(start.id)
+    assert ok is True
+

firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py

@@ -0,0 +1,189 @@
+import os
+import asyncio
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.types import ScrapeOptions
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_start_and_status():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
+    job_id = start.id
+
+    deadline = asyncio.get_event_loop().time() + 180
+    status = await client.get_crawl_status(job_id)
+    while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_crawl_status(job_id)
+
+    assert status.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_all_params():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    # rich scrape options including json format
+    json_schema = {
+        "type": "object",
+        "properties": {"title": {"type": "string"}},
+        "required": ["title"],
+    }
+    status = await client.crawl(
+        url="https://docs.firecrawl.dev",
+        prompt="Extract docs and blog",
+        include_paths=["/docs/*", "/blog/*"],
+        exclude_paths=["/admin/*"],
+        max_discovery_depth=2,
+        ignore_sitemap=False,
+        ignore_query_parameters=True,
+        limit=5,
+        crawl_entire_domain=False,
+        allow_external_links=True,
+        allow_subdomains=True,
+        delay=1,
+        max_concurrency=2,
+        integration="_e2e-test",
+        webhook="https://example.com/hook",
+        scrape_options=ScrapeOptions(
+            formats=[
+                "markdown",
+                "rawHtml",
+                {"type": "json", "prompt": "Extract title", "schema": json_schema},
+            ],
+            only_main_content=True,
+            mobile=False,
+            timeout=20000,
+            wait_for=500,
+            skip_tls_verification=False,
+            remove_base64_images=False,
+        ),
+        zero_data_retention=False,
+        poll_interval=2,
+        timeout=180,
+    )
+    assert status.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_start_crawl_with_options():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=5, max_discovery_depth=2)
+    assert start.id is not None and start.url is not None
+
+
+@pytest.mark.asyncio
+async def test_async_start_crawl_with_prompt():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://firecrawl.dev", prompt="Extract all blog posts", limit=3)
+    assert start.id is not None and start.url is not None
+
+
+@pytest.mark.asyncio
+async def test_async_get_crawl_status_shape():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
+    status = await client.get_crawl_status(start.id)
+    assert status.status in ("scraping", "completed", "failed")
+    assert status.completed >= 0
+    assert status.expires_at is not None
+    assert isinstance(status.data, list)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_wait():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    job = await client.crawl(url="https://docs.firecrawl.dev", limit=3, max_discovery_depth=2, poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_prompt_and_wait():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    job = await client.crawl(url="https://docs.firecrawl.dev", prompt="Extract all blog posts", limit=3, poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_scrape_options():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    scrape_opts = ScrapeOptions(formats=["markdown", "links"], only_main_content=False, mobile=True)
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
+    assert start.id is not None
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_with_json_format_object():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    scrape_opts = ScrapeOptions(formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}])
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
+    assert start.id is not None
+
+
+@pytest.mark.asyncio
+async def test_async_cancel_crawl():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
+    cancelled = await client.cancel_crawl(start.id)
+    assert cancelled is True
+
+
+@pytest.mark.asyncio
+async def test_async_get_crawl_errors_and_invalid_job():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
+    errs = await client.get_crawl_errors(start.id)
+    assert hasattr(errs, "errors") and hasattr(errs, "robots_blocked")
+    with pytest.raises(Exception):
+        await client.get_crawl_errors("invalid-job-id-12345")
+
+
+@pytest.mark.asyncio
+async def test_async_active_crawls():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    resp = await client.active_crawls()
+    assert hasattr(resp, "success") and hasattr(resp, "crawls")
+
+
+@pytest.mark.asyncio
+async def test_async_active_crawls_with_running_crawl():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
+    # fetch active crawls and assert our ID is listed
+    active = await client.active_crawls()
+    ids = [c.id for c in active.crawls]
+    assert start.id in ids
+    # cleanup
+    await client.cancel_crawl(start.id)
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_params_preview():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    params = await client.crawl_params_preview(
+        url="https://docs.firecrawl.dev",
+        prompt="Extract all blog posts and documentation",
+    )
+    assert params is not None
+    # basic sanity: at least one field should be suggested
+    has_any = any([
+        getattr(params, "limit", None) is not None,
+        getattr(params, "include_paths", None) is not None,
+        getattr(params, "max_discovery_depth", None) is not None,
+    ])
+    assert has_any
+
+

firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py

@@ -0,0 +1,39 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_extract_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    res = await client.extract(urls=["https://docs.firecrawl.dev"], prompt="Extract title")
+    assert res is not None
+
+
+@pytest.mark.asyncio
+async def test_async_extract_with_schema_and_options():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    res = await client.extract(
+        urls=["https://docs.firecrawl.dev"],
+        prompt="Extract title",
+        schema=schema,
+        system_prompt="You are a helpful extractor",
+        allow_external_links=False,
+        enable_web_search=False,
+        show_sources=False,
+        integration="_e2e-test",
+        # agent={"model": "FIRE-1", "prompt": "Extract title"}, # Skipping agent test in CI
+    )
+    assert res is not None
+

firecrawl/__tests__/e2e/v2/aio/test_aio_map.py

@@ -0,0 +1,41 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_map_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    resp = await client.map("https://docs.firecrawl.dev")
+    assert hasattr(resp, "links") and isinstance(resp.links, list)
+    if resp.links:
+        first = resp.links[0]
+        assert hasattr(first, "url") and isinstance(first.url, str) and first.url.startswith("http")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sitemap", ["only", "include", "skip"])
+async def test_async_map_with_all_params(sitemap):
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    resp = await client.map(
+        "https://docs.firecrawl.dev",
+        search="docs",
+        include_subdomains=True,
+        limit=10,
+        sitemap=sitemap,
+        timeout=15000,
+        integration="_e2e-test",
+    )
+    assert hasattr(resp, "links") and isinstance(resp.links, list)
+    assert len(resp.links) <= 10
+

firecrawl/__tests__/e2e/v2/aio/test_aio_scrape.py

@@ -0,0 +1,138 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.types import Document
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape("https://docs.firecrawl.dev")
+    assert isinstance(doc, Document)
+    assert (
+        (doc.markdown and len(doc.markdown) > 0)
+        or (doc.html and len(doc.html) > 0)
+        or (doc.raw_html and len(doc.raw_html) > 0)
+        or (doc.links is not None)
+        or (doc.screenshot is not None)
+        or (doc.json is not None)
+        or (doc.summary is not None)
+    )
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_all_params():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    # Include multiple formats with configuration
+    json_schema = {
+        "type": "object",
+        "properties": {"title": {"type": "string"}},
+        "required": ["title"],
+    }
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=[
+            "markdown",
+            "rawHtml",
+            {"type": "screenshot", "full_page": False, "quality": 70},
+            {"type": "json", "prompt": "Extract title", "schema": json_schema},
+        ],
+        headers={"User-Agent": "E2E-AIO"},
+        include_tags=["main"],
+        exclude_tags=["nav"],
+        only_main_content=True,
+        timeout=20000,
+        wait_for=500,
+        mobile=False,
+        parsers=["pdf"],
+        actions=[],
+        skip_tls_verification=False,
+        remove_base64_images=False,
+        fast_mode=False,
+        use_mock=None,
+        block_ads=False,
+        proxy="basic",
+        max_age=0,
+        store_in_cache=False,
+        integration="_e2e-test",
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_options_markdown():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=["markdown"],
+        only_main_content=False,
+        mobile=False,
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_screenshot_action_viewport():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=[{"type": "screenshot", "full_page": False, "quality": 80, "viewport": {"width": 800, "height": 600}}],
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fmt,expect_field", [
+    ("markdown", "markdown"),
+    ("html", "html"),
+    ("raw_html", "raw_html"),
+    ("links", "links"),
+    ("screenshot", "screenshot"),
+    ("summary", "summary"),
+])
+async def test_async_scrape_basic_formats(fmt, expect_field):
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape("https://docs.firecrawl.dev", formats=[fmt])
+    assert isinstance(doc, Document)
+    if expect_field == "markdown":
+        assert doc.markdown is not None
+    elif expect_field == "html":
+        assert doc.html is not None
+    elif expect_field == "raw_html":
+        assert doc.raw_html is not None
+    elif expect_field == "links":
+        assert isinstance(doc.links, list)
+    elif expect_field == "screenshot":
+        assert doc.screenshot is not None
+    elif expect_field == "summary":
+        assert doc.summary is not None
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_with_json_format_object():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    doc = await client.scrape(
+        "https://docs.firecrawl.dev",
+        formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],
+        only_main_content=True,
+    )
+    assert isinstance(doc, Document)
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_invalid_url():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    with pytest.raises(ValueError):
+        await client.scrape("")
+