firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/__tests__/e2e/v2/aio/test_aio_search.py

@@ -0,0 +1,183 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.types import ScrapeOptions, ScrapeFormats, SearchData, SearchResult, Document
+
+
+def _collect_texts(entries):
+    texts = []
+    for r in entries or []:
+        title = getattr(r, 'title', None) if hasattr(r, 'title') else None
+        desc = getattr(r, 'description', None) if hasattr(r, 'description') else None
+        if title:
+            texts.append(str(title).lower())
+        if desc:
+            texts.append(str(desc).lower())
+    return texts
+
+def _is_document(entry) -> bool:
+    try:
+        from firecrawl.v2.types import Document
+        return isinstance(entry, Document) or \
+               hasattr(entry, 'markdown') or \
+               hasattr(entry, 'html') or \
+               hasattr(entry, 'raw_html') or \
+               hasattr(entry, 'json') or \
+               hasattr(entry, 'screenshot') or \
+               hasattr(entry, 'change_tracking') or \
+               hasattr(entry, 'summary')
+    except Exception:
+        return hasattr(entry, 'markdown') or \
+               hasattr(entry, 'html') or \
+               hasattr(entry, 'raw_html') or \
+               hasattr(entry, 'json') or \
+               hasattr(entry, 'screenshot') or \
+               hasattr(entry, 'change_tracking') or \
+               hasattr(entry, 'summary')
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_search_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    data = await client.search("What is the capital of France?")
+    # Assert sections like sync tests
+    assert hasattr(data, "web")
+    assert hasattr(data, "news")
+    assert hasattr(data, "images")
+    assert data.web is not None
+    assert len(data.web) > 0
+    titles = [getattr(r, "title", None) for r in data.web]
+    descs = [getattr(r, "description", None) for r in data.web]
+    all_text = " ".join([t.lower() for t in titles if t] + [d.lower() for d in descs if d])
+    assert "paris" in all_text
+    assert data.news is None
+    assert data.images is None
+
+
+@pytest.mark.asyncio
+async def test_async_search_with_sources_and_limit():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    data = await client.search("firecrawl", sources=["web", "news"], limit=3)
+    # Sections present
+    assert hasattr(data, "web") and hasattr(data, "news") and hasattr(data, "images")
+    # Web present, images absent, news optional but if present respects limit
+    if data.web is not None:
+        assert len(data.web) <= 3
+    if data.news is not None:
+        assert len(data.news) <= 3
+    assert data.images is None
+
+
+@pytest.mark.asyncio
+async def test_async_search_with_all_params():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    data = await client.search(
+        "artificial intelligence",
+        sources=["web", "news"],
+        limit=3,
+        tbs="qdr:w",
+        location="US",
+        ignore_invalid_urls=False,
+        timeout=30000,
+        scrape_options={
+            "formats": ["markdown"],
+            "headers": {"User-Agent": "E2E-AIO"},
+            "include_tags": ["h1"],
+            "exclude_tags": ["nav"],
+            "only_main_content": False,
+            "timeout": 15000,
+            "wait_for": 2000,
+            "mobile": True,
+            "skip_tls_verification": True,
+            "remove_base64_images": False,
+        },
+    )
+    # Structure and type assertions mirroring sync
+    assert isinstance(data, SearchData)
+    assert hasattr(data, "web") and hasattr(data, "news") and hasattr(data, "images")
+    assert data.web is not None
+    assert len(data.web) <= 3
+    non_doc = [r for r in (data.web or []) if not _is_document(r)]
+    if non_doc:
+        combined = " ".join(_collect_texts(non_doc))
+        ai_terms = ["artificial", "intelligence", "ai", "machine", "learning"]
+        assert any(term in combined for term in ai_terms)
+    for r in data.web:
+        assert isinstance(r, (SearchResult, Document))
+        if isinstance(r, Document):
+            assert (r.markdown is not None) or (r.html is not None)
+        else:
+            assert hasattr(r, "url")
+            assert isinstance(r.url, str) and r.url.startswith("http")
+    if data.news is not None:
+        assert len(data.news) <= 10
+        for r in data.news:
+            assert isinstance(r, (SearchResult, Document))
+            if isinstance(r, Document):
+                assert (r.markdown is not None) or (r.html is not None)
+            else:
+                assert isinstance(r.url, str) and r.url.startswith("http")
+    assert data.images is None
+
+
+@pytest.mark.asyncio
+async def test_async_search_minimal_content_check():
+    """Stronger assertion similar to sync: content check on a known query."""
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    data = await client.search("What is the capital of France?")
+    assert hasattr(data, "web") and data.web is not None
+    non_doc = [r for r in (data.web or []) if not _is_document(r)]
+    if non_doc:
+        combined = " ".join(_collect_texts(non_doc))
+        assert "paris" in combined
+
+
+@pytest.mark.asyncio
+async def test_async_search_result_structure():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    data = await client.search("test query", limit=1)
+    if data.web and len(data.web) > 0:
+        result = data.web[0]
+        assert hasattr(result, "url")
+        assert hasattr(result, "title")
+        assert hasattr(result, "description")
+        assert isinstance(result.url, str) and result.url.startswith("http")
+        assert isinstance(getattr(result, "title", None), (str, type(None)))
+        assert isinstance(getattr(result, "description", None), (str, type(None)))
+
+
+@pytest.mark.asyncio
+async def test_async_search_formats_flexibility():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    # list string
+    res1 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=["markdown"]))
+    # list objects
+    res2 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=[{"type": "markdown"}]))
+    # ScrapeFormats object
+    res3 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=ScrapeFormats(markdown=True)))
+    assert isinstance(res1, SearchData) and hasattr(res1, "web")
+    assert isinstance(res2, SearchData) and hasattr(res2, "web")
+    assert isinstance(res3, SearchData) and hasattr(res3, "web")
+
+
+@pytest.mark.asyncio
+async def test_async_search_json_format_object():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    data = await client.search(
+        "site:docs.firecrawl.dev",
+        limit=1,
+        scrape_options={"formats": [{"type": "json", "prompt": "Extract page title", "schema": json_schema}]},
+    )
+    assert hasattr(data, "web")
+

firecrawl/__tests__/e2e/v2/aio/test_aio_usage.py

@@ -0,0 +1,35 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_get_concurrency():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    conc = await client.get_concurrency()
+    assert hasattr(conc, "concurrency") and hasattr(conc, "max_concurrency")
+
+
+@pytest.mark.asyncio
+async def test_async_get_credit_usage():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    credits = await client.get_credit_usage()
+    assert hasattr(credits, "remaining_credits")
+
+
+@pytest.mark.asyncio
+async def test_async_get_token_usage():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    tokens = await client.get_token_usage()
+    assert hasattr(tokens, "remaining_tokens")
+

firecrawl/__tests__/e2e/v2/aio/test_aio_watcher.py

@@ -0,0 +1,43 @@
+import os
+import asyncio
+import pytest
+from dotenv import load_dotenv
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.watcher_async import AsyncWatcher
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_watcher_crawl_progresses():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
+    statuses = []
+    async for snapshot in AsyncWatcher(client, start.id, kind="crawl", timeout=180):
+        statuses.append(snapshot.status)
+        if snapshot.status in ("completed", "failed"):
+            break
+    assert statuses and statuses[-1] in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_watcher_batch_progresses():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    statuses = []
+    async for snapshot in AsyncWatcher(client, start.id, kind="batch", timeout=240):
+        statuses.append(snapshot.status)
+        if snapshot.status in ("completed", "failed", "cancelled"):
+            break
+    assert statuses and statuses[-1] in ("completed", "failed", "cancelled")
+

firecrawl/__tests__/e2e/v2/conftest.py

@@ -0,0 +1,73 @@
+import os
+import json
+import pytest
+import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def _idmux(identity_request: dict) -> dict:
+    idmux_url = os.getenv("IDMUX_URL")
+    if not idmux_url:
+        raise EnvironmentError("IDMUX_URL is not set. E2E tests must use idmux for credentials.")
+    run_number = int(os.getenv("GITHUB_RUN_NUMBER") or 0)
+    payload = {
+        "refName": os.getenv("GITHUB_REF_NAME") or "local",
+        "runNumber": run_number,
+        "concurrency": identity_request.get("concurrency", 100),
+        **identity_request,
+    }
+    resp = requests.post(idmux_url + "/", json=payload)
+    resp.raise_for_status()
+    return resp.json()
+
+@pytest.fixture(scope="session")
+def api_url():
+    # Prefer TEST_URL, then FIRECRAWL_API_URL (for parity with JS), then legacy API_URL
+    return (
+        os.getenv("TEST_URL")
+        or os.getenv("FIRECRAWL_API_URL")
+        or os.getenv("API_URL")
+        or "https://api.firecrawl.dev"
+    )
+
+# Resolve identity and export environment at import time so tests that read env at module import succeed
+_IDENTITY = None
+_API_URL = (
+    os.getenv("TEST_URL")
+    or os.getenv("FIRECRAWL_API_URL")
+    or os.getenv("API_URL")
+    or "https://api.firecrawl.dev"
+)
+
+_IDMUX_URL = os.getenv("IDMUX_URL")
+if _IDMUX_URL:
+    run_name = os.getenv("PYTEST_RUN_NAME") or "py-e2e"
+    # If IDMUX_URL is set, idmux MUST succeed; do not silently fall back
+    _IDENTITY = _idmux({"name": run_name})
+    os.environ["API_KEY"] = _IDENTITY.get("apiKey", "")
+    os.environ["API_URL"] = _API_URL
+
+@pytest.fixture(scope="session")
+def api_identity():
+    return _IDENTITY or {"apiKey": os.getenv("API_KEY") or "", "teamId": os.getenv("TEST_TEAM_ID") or os.getenv("TEAM_ID") or ""}
+
+@pytest.fixture(autouse=True)
+def _inject_client(request, api_identity, api_url):
+    # For class-based tests that rely on self.client, inject a client if missing
+    inst = getattr(request, "instance", None)
+    if inst is not None and not hasattr(inst, "client"):
+        try:
+            from firecrawl import Firecrawl
+            inst.client = Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url)
+        except Exception:
+            pass
+    # For function-based modules that expect a module-level `firecrawl` symbol
+    mod = getattr(request, "module", None)
+    if mod is not None and not hasattr(mod, "firecrawl"):
+        try:
+            from firecrawl import Firecrawl
+            setattr(mod, "firecrawl", Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url))
+        except Exception:
+            pass
+

firecrawl/__tests__/e2e/v2/test_async.py

@@ -0,0 +1,73 @@
+import os
+import asyncio
+import pytest
+from dotenv import load_dotenv
+
+from firecrawl import AsyncFirecrawl
+from firecrawl.v2.types import Document
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+@pytest.mark.asyncio
+async def test_async_scrape_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    doc = await client.scrape("https://docs.firecrawl.dev")
+    assert isinstance(doc, Document)
+    # Accept any primary content or alternate outputs
+    assert doc.markdown is not None and doc.markdown and len(doc.markdown) > 0
+
+
+@pytest.mark.asyncio
+async def test_async_crawl_start_and_status():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
+    job_id = start.id
+
+    # Poll status until terminal or timeout
+    deadline = asyncio.get_event_loop().time() + 180
+    status = await client.get_crawl_status(job_id)
+    while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_crawl_status(job_id)
+
+    assert status.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_start_and_status():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    job_id = start.id
+
+    deadline = asyncio.get_event_loop().time() + 240
+    status = await client.get_batch_scrape_status(job_id)
+    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_batch_scrape_status(job_id)
+
+    assert status.status in ("completed", "failed", "cancelled")
+
+
+@pytest.mark.asyncio
+async def test_async_usage_minimal():
+    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+    conc = await client.get_concurrency()
+    assert hasattr(conc, "concurrency") and hasattr(conc, "max_concurrency")
+
+    credits = await client.get_credit_usage()
+    assert hasattr(credits, "remaining_credits")
+
+    tokens = await client.get_token_usage()
+    assert hasattr(tokens, "remaining_tokens")
+

firecrawl/__tests__/e2e/v2/test_batch_scrape.py

@@ -0,0 +1,105 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import Firecrawl
+from firecrawl.v2.types import ScrapeOptions
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+    raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+    raise ValueError("API_URL is not set")
+
+
+class TestBatchScrapeE2E:
+    """End-to-end tests for batch scrape (v2)."""
+
+    def setup_method(self):
+        self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+
+    def test_batch_scrape_minimal(self):
+        """Start a small batch and wait for completion."""
+        urls = [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ]
+
+        job = self.client.batch_scrape(urls, formats=["markdown"], poll_interval=1, wait_timeout=120)
+
+        assert job.status in ["completed", "failed"]
+        assert job.completed >= 0
+        assert job.total >= 0
+        assert isinstance(job.data, list)
+
+    def test_start_batch_minimal_and_status(self):
+        """Start via start_batch_scrape (minimal), then fetch status once."""
+        urls = [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ]
+
+        start_resp = self.client.start_batch_scrape(urls, formats=["markdown"], ignore_invalid_urls=True)
+        assert start_resp.id is not None
+        assert start_resp.url is not None
+
+        job = self.client.get_batch_scrape_status(start_resp.id)
+        assert job.status in ["scraping", "completed", "failed"]
+        assert job.total >= 0
+
+    def test_wait_batch_with_all_params(self):
+        """Blocking waiter with JSON and changeTracking formats plus many options."""
+        urls = [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ]
+
+        json_schema = {
+            "type": "object",
+            "properties": {
+                "title": {"type": "string"}
+            },
+            "required": ["title"],
+        }
+
+        opts = ScrapeOptions(
+            formats=[
+                "markdown",
+                {"type": "json", "prompt": "Extract page title", "schema": json_schema},
+                {"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
+            ],
+            only_main_content=True,
+            mobile=False,
+        )
+
+        job = self.client.batch_scrape(
+            urls,
+            formats=opts.formats,
+            only_main_content=opts.only_main_content,
+            mobile=opts.mobile,
+            ignore_invalid_urls=True,
+            max_concurrency=2,
+            zero_data_retention=False,
+            poll_interval=1,
+            wait_timeout=180,
+        )
+
+        assert job.status in ["completed", "failed"]
+        assert job.completed >= 0
+        assert job.total >= 0
+        assert isinstance(job.data, list)
+
+    def test_cancel_batch(self):
+        """Start a batch and cancel it."""
+        urls = [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ]
+
+        start_resp = self.client.start_batch_scrape(urls, formats=["markdown"], max_concurrency=1)
+        assert start_resp.id is not None
+
+        cancelled = self.client.cancel_batch_scrape(start_resp.id)
+        assert cancelled is True
+