firecrawl-py 3.3.1__py3-none-any.whl → 3.3.3__py3-none-any.whl

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_map.py

@@ -1,40 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_map_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    resp = await client.map("https://docs.firecrawl.dev")
-    assert hasattr(resp, "links") and isinstance(resp.links, list)
-    if resp.links:
-        first = resp.links[0]
-        assert hasattr(first, "url") and isinstance(first.url, str) and first.url.startswith("http")
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("sitemap", ["only", "include", "skip"])
-async def test_async_map_with_all_params(sitemap):
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    resp = await client.map(
-        "https://docs.firecrawl.dev",
-        search="docs",
-        include_subdomains=True,
-        limit=10,
-        sitemap=sitemap,
-        timeout=15000,
-    )
-    assert hasattr(resp, "links") and isinstance(resp.links, list)
-    assert len(resp.links) <= 10
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_scrape.py

@@ -1,137 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-from firecrawl.v2.types import Document
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    doc = await client.scrape("https://docs.firecrawl.dev")
-    assert isinstance(doc, Document)
-    assert (
-        (doc.markdown and len(doc.markdown) > 0)
-        or (doc.html and len(doc.html) > 0)
-        or (doc.raw_html and len(doc.raw_html) > 0)
-        or (doc.links is not None)
-        or (doc.screenshot is not None)
-        or (doc.json is not None)
-        or (doc.summary is not None)
-    )
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_with_all_params():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    # Include multiple formats with configuration
-    json_schema = {
-        "type": "object",
-        "properties": {"title": {"type": "string"}},
-        "required": ["title"],
-    }
-    doc = await client.scrape(
-        "https://docs.firecrawl.dev",
-        formats=[
-            "markdown",
-            "rawHtml",
-            {"type": "screenshot", "full_page": False, "quality": 70},
-            {"type": "json", "prompt": "Extract title", "schema": json_schema},
-        ],
-        headers={"User-Agent": "E2E-AIO"},
-        include_tags=["main"],
-        exclude_tags=["nav"],
-        only_main_content=True,
-        timeout=20000,
-        wait_for=500,
-        mobile=False,
-        parsers=["pdf"],
-        actions=[],
-        skip_tls_verification=False,
-        remove_base64_images=False,
-        fast_mode=False,
-        use_mock=None,
-        block_ads=False,
-        proxy="basic",
-        max_age=0,
-        store_in_cache=False,
-    )
-    assert isinstance(doc, Document)
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_with_options_markdown():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    doc = await client.scrape(
-        "https://docs.firecrawl.dev",
-        formats=["markdown"],
-        only_main_content=False,
-        mobile=False,
-    )
-    assert isinstance(doc, Document)
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_with_screenshot_action_viewport():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    doc = await client.scrape(
-        "https://docs.firecrawl.dev",
-        formats=[{"type": "screenshot", "full_page": False, "quality": 80, "viewport": {"width": 800, "height": 600}}],
-    )
-    assert isinstance(doc, Document)
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("fmt,expect_field", [
-    ("markdown", "markdown"),
-    ("html", "html"),
-    ("raw_html", "raw_html"),
-    ("links", "links"),
-    ("screenshot", "screenshot"),
-    ("summary", "summary"),
-])
-async def test_async_scrape_basic_formats(fmt, expect_field):
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    doc = await client.scrape("https://docs.firecrawl.dev", formats=[fmt])
-    assert isinstance(doc, Document)
-    if expect_field == "markdown":
-        assert doc.markdown is not None
-    elif expect_field == "html":
-        assert doc.html is not None
-    elif expect_field == "raw_html":
-        assert doc.raw_html is not None
-    elif expect_field == "links":
-        assert isinstance(doc.links, list)
-    elif expect_field == "screenshot":
-        assert doc.screenshot is not None
-    elif expect_field == "summary":
-        assert doc.summary is not None
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_with_json_format_object():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    doc = await client.scrape(
-        "https://docs.firecrawl.dev",
-        formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],
-        only_main_content=True,
-    )
-    assert isinstance(doc, Document)
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_invalid_url():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    with pytest.raises(ValueError):
-        await client.scrape("")
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py

@@ -1,248 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-from firecrawl.types import (
-    SearchData,
-    Document,
-    ScrapeOptions,
-    ScrapeFormats,
-    SearchResultWeb,
-    SearchResultNews,
-    SearchResultImages,
-)
-
-load_dotenv()
-
-firecrawl = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-
-def _collect_texts(entries):
-    texts = []
-    for r in entries or []:
-        title = getattr(r, 'title', None) if hasattr(r, 'title') else None
-        desc = getattr(r, 'description', None) if hasattr(r, 'description') else None
-        if title:
-            texts.append(str(title).lower())
-        if desc:
-            texts.append(str(desc).lower())
-    return texts
-
-def _is_document(entry) -> bool:
-    try:
-        from firecrawl.v2.types import Document
-        return isinstance(entry, Document) or \
-               hasattr(entry, 'markdown') or \
-               hasattr(entry, 'html') or \
-               hasattr(entry, 'raw_html') or \
-               hasattr(entry, 'json') or \
-               hasattr(entry, 'screenshot') or \
-               hasattr(entry, 'change_tracking') or \
-               hasattr(entry, 'summary')
-    except Exception:
-        return hasattr(entry, 'markdown') or \
-               hasattr(entry, 'html') or \
-               hasattr(entry, 'raw_html') or \
-               hasattr(entry, 'json') or \
-               hasattr(entry, 'screenshot') or \
-               hasattr(entry, 'change_tracking') or \
-               hasattr(entry, 'summary')
-
-@pytest.mark.asyncio
-async def test_async_search_minimal_request():
-    results = await firecrawl.search(
-        query="What is the capital of France?"
-    )
-    assert isinstance(results, SearchData)
-    assert hasattr(results, 'web')
-    assert results.web is not None
-    assert len(results.web) > 0
-    assert hasattr(results, 'news')
-    assert results.news is None
-    assert hasattr(results, 'images')
-    assert results.images is None
-
-    for result in results.web:
-        assert isinstance(result, SearchResultWeb)
-        assert hasattr(result, 'url')
-        assert hasattr(result, 'title')
-        assert hasattr(result, 'description')
-        assert result.url.startswith('http')
-        assert result.title is not None
-        assert result.description is not None
-
-    all_text = ' '.join(_collect_texts(results.web))
-    assert 'paris' in all_text
-
-    assert results.news is None
-    assert results.images is None
-
-@pytest.mark.asyncio
-async def test_async_search_with_sources():
-    results = await firecrawl.search(
-        query="firecrawl",
-        sources=["web", "news", "images"],
-        limit=3
-    )
-    assert isinstance(results, SearchData)
-    assert results.web is not None
-    assert len(results.web) <= 3
-    assert isinstance(results.web[0], SearchResultWeb)
-
-    if results.news is not None:
-        assert len(results.news) <= 3
-        assert isinstance(results.news[0], SearchResultNews)
-
-    if results.images is not None:
-        assert len(results.images) <= 3
-        assert isinstance(results.images[0], SearchResultImages)
-
-    web_titles = [result.title.lower() for result in results.web]
-    web_descriptions = [result.description.lower() for result in results.web]
-    all_web_text = ' '.join(web_titles + web_descriptions)
-    assert 'firecrawl' in all_web_text
-
-@pytest.mark.asyncio
-async def test_async_search_result_structure():
-    results = await firecrawl.search(
-        query="test query",
-        limit=1
-    )
-    if results.web and len(results.web) > 0:
-        result = results.web[0]
-        assert hasattr(result, 'url')
-        assert hasattr(result, 'title')
-        assert hasattr(result, 'description')
-        assert isinstance(result.url, str)
-        assert isinstance(result.title, str) or result.title is None
-        assert isinstance(result.description, str) or result.description is None
-        assert result.url.startswith('http')
-
-@pytest.mark.asyncio
-async def test_async_search_all_parameters():
-    from firecrawl.types import ScrapeOptions, Location, WaitAction
-    schema = {
-        "type": "object",
-        "properties": {
-            "title": {"type": "string"},
-            "description": {"type": "string"},
-            "url": {"type": "string"}
-        },
-        "required": ["title", "description"]
-    }
-    results = await firecrawl.search(
-        query="artificial intelligence",
-        sources=[
-            {"type": "web"},
-            {"type": "news"}
-        ],
-        limit=3,
-        tbs="qdr:m",
-        location="US",
-        ignore_invalid_urls=True,
-        timeout=60000,
-        scrape_options=ScrapeOptions(
-            formats=[
-                "markdown",
-                "html",
-                {
-                    "type": "json",
-                    "prompt": "Extract the title and description from the page",
-                    "schema": schema
-                },
-                {"type": "summary"}
-            ],
-            headers={"User-Agent": "Firecrawl-Test/1.0"},
-            include_tags=["h1", "h2", "p"],
-            exclude_tags=["nav", "footer"],
-            only_main_content=True,
-            wait_for=2000,
-            mobile=False,
-            skip_tls_verification=False,
-            remove_base64_images=True,
-            block_ads=True,
-            proxy="basic",
-            max_age=3600000,
-            store_in_cache=True,
-            location=Location(
-                country="US",
-                languages=["en"]
-            ),
-            actions=[
-                WaitAction(milliseconds=1000)
-            ]
-        )
-    )
-    assert isinstance(results, SearchData)
-    assert hasattr(results, 'web')
-    assert hasattr(results, 'news')
-    assert hasattr(results, 'images')
-    assert results.web is not None
-    assert len(results.web) <= 3
-
-    non_doc_entries = [r for r in (results.web or []) if not _is_document(r)]
-    if non_doc_entries:
-        all_web_text = ' '.join(_collect_texts(non_doc_entries))
-        ai_terms = ['artificial', 'intelligence', 'ai', 'machine', 'learning']
-        assert any(term in all_web_text for term in ai_terms)
-
-    for result in results.web:
-        assert isinstance(result, (SearchResultWeb, Document))
-        if isinstance(result, Document):
-            assert (result.markdown is not None) or (result.html is not None)
-        else:
-            assert hasattr(result, 'url')
-            assert isinstance(result.url, str) and result.url.startswith('http')
-
-    if results.news is not None:
-        assert len(results.news) <= 3
-        for result in results.news:
-            assert isinstance(result, (SearchResultNews, Document))
-            if isinstance(result, Document):
-                assert (result.markdown is not None) or (result.html is not None)
-            else:
-                assert hasattr(result, 'url')
-                assert isinstance(result.url, str) and result.url.startswith('http')
-
-    assert results.images is None
-
-@pytest.mark.asyncio
-async def test_async_search_formats_flexibility():
-    # Test with list format
-    results1 = await firecrawl.search(
-        query="python programming",
-        limit=1,
-        scrape_options=ScrapeOptions(
-            formats=["markdown"]
-        )
-    )
-    # Test with ScrapeFormats object
-    results2 = await firecrawl.search(
-        query="python programming",
-        limit=1,
-        scrape_options=ScrapeOptions(
-            formats=ScrapeFormats(markdown=True)
-        )
-    )
-    assert isinstance(results1, SearchData)
-    assert isinstance(results2, SearchData)
-    assert results1.web is not None
-    assert results2.web is not None
-
-@pytest.mark.asyncio
-async def test_async_search_with_json_format_object():
-    json_schema = {
-        "type": "object",
-        "properties": {
-            "title": {"type": "string"}
-        },
-        "required": ["title"],
-    }
-    results = await firecrawl.search(
-        query="site:docs.firecrawl.dev",
-        limit=1,
-        scrape_options=ScrapeOptions(
-            formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
-        ),
-    )
-    assert isinstance(results, SearchData)
-    assert results.web is not None and len(results.web) >= 0

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_usage.py

@@ -1,35 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_get_concurrency():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    conc = await client.get_concurrency()
-    assert hasattr(conc, "concurrency") and hasattr(conc, "max_concurrency")
-
-
-@pytest.mark.asyncio
-async def test_async_get_credit_usage():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    credits = await client.get_credit_usage()
-    assert hasattr(credits, "remaining_credits")
-
-
-@pytest.mark.asyncio
-async def test_async_get_token_usage():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    tokens = await client.get_token_usage()
-    assert hasattr(tokens, "remaining_tokens")
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_watcher.py

@@ -1,43 +0,0 @@
-import os
-import asyncio
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-from firecrawl.v2.watcher_async import AsyncWatcher
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_watcher_crawl_progresses():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
-    statuses = []
-    async for snapshot in AsyncWatcher(client, start.id, kind="crawl", timeout=180):
-        statuses.append(snapshot.status)
-        if snapshot.status in ("completed", "failed"):
-            break
-    assert statuses and statuses[-1] in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_watcher_batch_progresses():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], max_concurrency=1)
-    statuses = []
-    async for snapshot in AsyncWatcher(client, start.id, kind="batch", timeout=240):
-        statuses.append(snapshot.status)
-        if snapshot.status in ("completed", "failed", "cancelled"):
-            break
-    assert statuses and statuses[-1] in ("completed", "failed", "cancelled")
-

build/lib/firecrawl/__tests__/e2e/v2/conftest.py

@@ -1,73 +0,0 @@
-import os
-import json
-import pytest
-import requests
-from dotenv import load_dotenv
-
-load_dotenv()
-
-def _idmux(identity_request: dict) -> dict:
-    idmux_url = os.getenv("IDMUX_URL")
-    if not idmux_url:
-        raise EnvironmentError("IDMUX_URL is not set. E2E tests must use idmux for credentials.")
-    run_number = int(os.getenv("GITHUB_RUN_NUMBER") or 0)
-    payload = {
-        "refName": os.getenv("GITHUB_REF_NAME") or "local",
-        "runNumber": run_number,
-        "concurrency": identity_request.get("concurrency", 100),
-        **identity_request,
-    }
-    resp = requests.post(idmux_url + "/", json=payload)
-    resp.raise_for_status()
-    return resp.json()
-
-@pytest.fixture(scope="session")
-def api_url():
-    # Prefer TEST_URL, then FIRECRAWL_API_URL (for parity with JS), then legacy API_URL
-    return (
-        os.getenv("TEST_URL")
-        or os.getenv("FIRECRAWL_API_URL")
-        or os.getenv("API_URL")
-        or "https://api.firecrawl.dev"
-    )
-
-# Resolve identity and export environment at import time so tests that read env at module import succeed
-_IDENTITY = None
-_API_URL = (
-    os.getenv("TEST_URL")
-    or os.getenv("FIRECRAWL_API_URL")
-    or os.getenv("API_URL")
-    or "https://api.firecrawl.dev"
-)
-
-_IDMUX_URL = os.getenv("IDMUX_URL")
-if _IDMUX_URL:
-    run_name = os.getenv("PYTEST_RUN_NAME") or "py-e2e"
-    # If IDMUX_URL is set, idmux MUST succeed; do not silently fall back
-    _IDENTITY = _idmux({"name": run_name})
-    os.environ["API_KEY"] = _IDENTITY.get("apiKey", "")
-    os.environ["API_URL"] = _API_URL
-
-@pytest.fixture(scope="session")
-def api_identity():
-    return _IDENTITY or {"apiKey": os.getenv("API_KEY") or "", "teamId": os.getenv("TEST_TEAM_ID") or os.getenv("TEAM_ID") or ""}
-
-@pytest.fixture(autouse=True)
-def _inject_client(request, api_identity, api_url):
-    # For class-based tests that rely on self.client, inject a client if missing
-    inst = getattr(request, "instance", None)
-    if inst is not None and not hasattr(inst, "client"):
-        try:
-            from firecrawl import Firecrawl
-            inst.client = Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url)
-        except Exception:
-            pass
-    # For function-based modules that expect a module-level `firecrawl` symbol
-    mod = getattr(request, "module", None)
-    if mod is not None and not hasattr(mod, "firecrawl"):
-        try:
-            from firecrawl import Firecrawl
-            setattr(mod, "firecrawl", Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url))
-        except Exception:
-            pass
-

build/lib/firecrawl/__tests__/e2e/v2/test_async.py

@@ -1,73 +0,0 @@
-import os
-import asyncio
-import pytest
-from dotenv import load_dotenv
-
-from firecrawl import AsyncFirecrawl
-from firecrawl.v2.types import Document
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_scrape_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    doc = await client.scrape("https://docs.firecrawl.dev")
-    assert isinstance(doc, Document)
-    # Accept any primary content or alternate outputs
-    assert doc.markdown is not None and doc.markdown and len(doc.markdown) > 0
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_start_and_status():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
-    job_id = start.id
-
-    # Poll status until terminal or timeout
-    deadline = asyncio.get_event_loop().time() + 180
-    status = await client.get_crawl_status(job_id)
-    while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
-        await asyncio.sleep(2)
-        status = await client.get_crawl_status(job_id)
-
-    assert status.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_start_and_status():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], max_concurrency=1)
-    job_id = start.id
-
-    deadline = asyncio.get_event_loop().time() + 240
-    status = await client.get_batch_scrape_status(job_id)
-    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
-        await asyncio.sleep(2)
-        status = await client.get_batch_scrape_status(job_id)
-
-    assert status.status in ("completed", "failed", "cancelled")
-
-
-@pytest.mark.asyncio
-async def test_async_usage_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    conc = await client.get_concurrency()
-    assert hasattr(conc, "concurrency") and hasattr(conc, "max_concurrency")
-
-    credits = await client.get_credit_usage()
-    assert hasattr(credits, "remaining_credits")
-
-    tokens = await client.get_token_usage()
-    assert hasattr(tokens, "remaining_tokens")
-