firecrawl 4.12.0__py3-none-any.whl

firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_params.py

@@ -0,0 +1,12 @@
+import pytest
+from firecrawl.v2.types import CrawlParamsRequest
+from firecrawl.v2.methods.aio import crawl as aio_crawl
+
+
+@pytest.mark.asyncio
+async def test_crawl_params_request_validation():
+    with pytest.raises(ValueError):
+        await aio_crawl.crawl_params_preview(None, CrawlParamsRequest(url="", prompt="x"))
+    with pytest.raises(ValueError):
+        await aio_crawl.crawl_params_preview(None, CrawlParamsRequest(url="https://x", prompt=""))
+

firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_request_preparation.py

@@ -0,0 +1,79 @@
+from firecrawl.v2.types import CrawlRequest, ScrapeOptions, WebhookConfig
+from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
+
+
+class TestAsyncCrawlRequestPreparation:
+    def test_basic_request(self):
+        req = CrawlRequest(url="https://example.com")
+        payload = _prepare_crawl_request(req)
+        assert payload["url"] == "https://example.com"
+
+    def test_field_mappings(self):
+        req = CrawlRequest(
+            url="https://example.com",
+            include_paths=["/docs/*"],
+            exclude_paths=["/admin/*"],
+            max_discovery_depth=2,
+            sitemap="skip",
+            ignore_query_parameters=True,
+            crawl_entire_domain=True,
+            allow_external_links=False,
+            allow_subdomains=True,
+            max_concurrency=5,
+            zero_data_retention=True,
+        )
+        payload = _prepare_crawl_request(req)
+        assert payload["includePaths"] == ["/docs/*"]
+        assert payload["excludePaths"] == ["/admin/*"]
+        assert payload["maxDiscoveryDepth"] == 2
+        assert payload["sitemap"] == "skip"
+        assert payload["ignoreQueryParameters"] is True
+        assert payload["crawlEntireDomain"] is True
+        assert payload["allowExternalLinks"] is False
+        assert payload["allowSubdomains"] is True
+        assert payload["maxConcurrency"] == 5
+        assert payload["zeroDataRetention"] is True
+
+    def test_webhook_preparation(self):
+        # string webhook
+        req = CrawlRequest(url="https://example.com", webhook="https://example.com/hook")
+        payload = _prepare_crawl_request(req)
+        assert payload["webhook"] == "https://example.com/hook"
+
+        # object webhook
+        req2 = CrawlRequest(url="https://example.com", webhook=WebhookConfig(url="https://x/h", headers={"X": "1"}, events=["completed"]))
+        payload2 = _prepare_crawl_request(req2)
+        assert isinstance(payload2["webhook"], dict)
+        assert payload2["webhook"]["url"] == "https://x/h"
+        assert payload2["webhook"]["headers"] == {"X": "1"}
+
+    def test_webhook_none_values_excluded(self):
+        req = CrawlRequest(
+            url="https://example.com",
+            webhook=WebhookConfig(url="https://example.com/webhook", headers=None, metadata=None, events=None),
+        )
+        payload = _prepare_crawl_request(req)
+        webhook = payload["webhook"]
+        assert webhook["url"] == "https://example.com/webhook"
+        assert "headers" not in webhook
+        assert "metadata" not in webhook
+        assert "events" not in webhook
+
+    def test_all_fields_including_integration(self):
+        req = CrawlRequest(
+            url="https://example.com",
+            include_paths=["/docs/*"],
+            exclude_paths=["/admin/*"],
+            max_discovery_depth=2,
+            sitemap="include",
+            ignore_query_parameters=True,
+            crawl_entire_domain=False,
+            allow_external_links=True,
+            allow_subdomains=True,
+            max_concurrency=3,
+            zero_data_retention=False,
+            integration="  _unit-test  ",
+        )
+        payload = _prepare_crawl_request(req)
+        assert payload["integration"] == "_unit-test"
+

firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_validation.py

@@ -0,0 +1,12 @@
+from firecrawl.v2.types import CrawlRequest, ScrapeOptions
+from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
+import pytest
+
+
+class TestAsyncCrawlValidation:
+    def test_invalid_url(self):
+        with pytest.raises(ValueError):
+            _prepare_crawl_request(CrawlRequest(url=""))
+        with pytest.raises(ValueError):
+            _prepare_crawl_request(CrawlRequest(url="   "))
+

firecrawl/__tests__/unit/v2/methods/aio/test_aio_map_request_preparation.py

@@ -0,0 +1,20 @@
+import pytest
+from firecrawl.v2.types import MapOptions
+from firecrawl.v2.methods.aio.map import _prepare_map_request
+
+
+class TestAsyncMapRequestPreparation:
+    def test_basic(self):
+        payload = _prepare_map_request("https://example.com")
+        assert payload["url"] == "https://example.com"
+
+    def test_fields(self):
+        opts = MapOptions(search="docs", include_subdomains=True, limit=10, sitemap="only", timeout=15000, integration="  _unit-test  ")
+        payload = _prepare_map_request("https://example.com", opts)
+        assert payload["search"] == "docs"
+        assert payload["includeSubdomains"] is True
+        assert payload["limit"] == 10
+        assert payload["sitemap"] == "only"
+        assert payload["timeout"] == 15000
+        assert payload["integration"] == "_unit-test"
+

firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py

@@ -0,0 +1,50 @@
+import pytest
+from firecrawl.v2.types import ScrapeOptions, Location
+from firecrawl.v2.methods.aio.scrape import _prepare_scrape_request
+
+
+class TestAsyncScrapeRequestPreparation:
+    @pytest.mark.asyncio
+    async def test_basic_request_preparation(self):
+        payload = await _prepare_scrape_request("https://example.com", None)
+        assert payload["url"] == "https://example.com"
+
+    @pytest.mark.asyncio
+    async def test_options_conversion(self):
+        opts = ScrapeOptions(
+            formats=["markdown", {"type": "screenshot", "full_page": True, "quality": 80}],
+            include_tags=["main"],
+            exclude_tags=["nav"],
+            only_main_content=True,
+            wait_for=500,
+            timeout=30000,
+            mobile=True,
+            parsers=["pdf"],
+            location=Location(country="us", languages=["en"]),
+            skip_tls_verification=False,
+            remove_base64_images=False,
+            fast_mode=True,
+            use_mock="test",
+            block_ads=False,
+            proxy="basic",
+            max_age=1000,
+            store_in_cache=False,
+        )
+        payload = await _prepare_scrape_request("https://example.com", opts)
+        assert payload["url"] == "https://example.com"
+        assert isinstance(payload.get("formats"), list) and "markdown" in payload["formats"]
+        assert payload["includeTags"] == ["main"]
+        assert payload["excludeTags"] == ["nav"]
+        assert payload["onlyMainContent"] is True
+        assert payload["waitFor"] == 500
+        assert payload["timeout"] == 30000
+        assert payload["mobile"] is True
+        assert payload["skipTlsVerification"] is False
+        assert payload["removeBase64Images"] is False
+        assert payload["fastMode"] is True
+        assert payload["useMock"] == "test"
+        assert payload["blockAds"] is False
+        assert payload["proxy"] == "basic"
+        assert payload["maxAge"] == 1000
+        assert payload["storeInCache"] is False
+

firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py

@@ -0,0 +1,64 @@
+import pytest
+from firecrawl.v2.types import SearchRequest, ScrapeOptions
+from firecrawl.v2.methods.aio.search import _prepare_search_request
+
+
+class TestAsyncSearchRequestPreparation:
+    def test_basic_request_preparation(self):
+        request = SearchRequest(query="test query")
+        data = _prepare_search_request(request)
+        assert data["query"] == "test query"
+        assert "ignore_invalid_urls" not in data
+        assert "scrape_options" not in data
+
+    def test_all_fields_conversion(self):
+        scrape_opts = ScrapeOptions(
+            formats=["markdown"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False,
+        )
+        request = SearchRequest(
+            query="test query",
+            sources=["web", "news"],
+            limit=10,
+            tbs="qdr:w",
+            location="US",
+            ignore_invalid_urls=False,
+            timeout=30000,
+            scrape_options=scrape_opts,
+            integration="  _unit-test  ",
+        )
+        data = _prepare_search_request(request)
+        assert data["ignoreInvalidURLs"] is False
+        assert "scrapeOptions" in data
+        assert data["integration"] == "_unit-test"
+
+    def test_exclude_none_behavior(self):
+        request = SearchRequest(
+            query="test",
+            sources=None,
+            limit=None,
+            tbs=None,
+            location=None,
+            ignore_invalid_urls=None,
+            timeout=None,
+            scrape_options=None,
+        )
+        data = _prepare_search_request(request)
+        assert "query" in data
+        assert len(data) == 1
+
+    def test_empty_scrape_options(self):
+        request = SearchRequest(query="test", scrape_options=ScrapeOptions())
+        data = _prepare_search_request(request)
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        assert "onlyMainContent" in scrape_data
+        assert "mobile" in scrape_data

firecrawl/__tests__/unit/v2/methods/aio/test_batch_request_preparation_async.py

@@ -0,0 +1,28 @@
+from firecrawl.v2.types import ScrapeOptions, Location
+from firecrawl.v2.methods.aio.batch import _prepare as _prepare_batch
+
+
+class TestAsyncBatchRequestPreparation:
+    def test_urls_validation_and_conversion(self):
+        payload = _prepare_batch(["https://example.com", "http://foo.bar"], options=None)
+        assert payload["urls"] == ["https://example.com", "http://foo.bar"]
+
+    def test_options_and_batch_fields(self):
+        opts = ScrapeOptions(formats=["markdown"], only_main_content=True)
+        payload = _prepare_batch(
+            ["https://example.com"],
+            options=opts,
+            webhook="https://hook.example",
+            append_to_id="00000000-0000-0000-0000-000000000000",
+            ignore_invalid_urls=True,
+            max_concurrency=3,
+            zero_data_retention=True,
+            integration="zapier",
+        )
+        assert payload["webhook"] == "https://hook.example"
+        assert payload["appendToId"] == "00000000-0000-0000-0000-000000000000"
+        assert payload["ignoreInvalidURLs"] is True
+        assert payload["maxConcurrency"] == 3
+        assert payload["zeroDataRetention"] is True
+        assert payload["integration"] == "zapier"
+

firecrawl/__tests__/unit/v2/methods/aio/test_ensure_async.py

@@ -0,0 +1,117 @@
+import asyncio
+import time
+import httpx
+import pytest
+
+from firecrawl.v2.client_async import AsyncFirecrawlClient
+from firecrawl.v2.utils.http_client_async import AsyncHttpClient
+from firecrawl.v2.utils.http_client import HttpClient
+from firecrawl.v2.methods.aio import batch as aio_batch
+
+
+@pytest.mark.asyncio
+async def test_scrape_concurrency(monkeypatch):
+    async def fake_post(self, endpoint, data, headers=None, timeout=None):
+        await asyncio.sleep(0.1)
+        return httpx.Response(200, json={"success": True, "data": {}})
+
+    monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    start = time.perf_counter()
+    await asyncio.gather(
+        client.scrape("https://firecrawl.dev"),
+        client.scrape("https://firecrawl.dev"),
+        client.scrape("https://firecrawl.dev")
+    )
+    elapsed = time.perf_counter() - start
+
+    # If calls run concurrently, total should be close to single 0.1s delay, not 0.3s
+    assert elapsed < 0.25
+
+
+@pytest.mark.asyncio
+async def test_event_loop_not_blocked(monkeypatch):
+    ticks = 0
+
+    async def ticker():
+        nonlocal ticks
+        for _ in range(5):
+            await asyncio.sleep(0.05)
+            ticks += 1
+
+    async def fake_post(self, endpoint, data, headers=None, timeout=None):
+        await asyncio.sleep(0.2)
+        return httpx.Response(200, json={"success": True, "data": {}})
+
+    monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    await asyncio.gather(ticker(), client.scrape("https://a"))
+    # If scrape awaited properly, ticker should have progressed several steps
+    assert ticks >= 3
+
+
+@pytest.mark.asyncio
+async def test_wait_batch_scrape_polling_interval(monkeypatch):
+    # Simulate one scraping status then completed
+    class S:  # simple status holder
+        def __init__(self, status):
+            self.status = status
+
+    states = ["scraping", "completed"]
+
+    async def fake_status(client, job_id):
+        state = states.pop(0)
+        return S(state)
+
+    monkeypatch.setattr(aio_batch, "get_batch_scrape_status", fake_status)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    start = time.perf_counter()
+    await client.wait_batch_scrape("job-1", poll_interval=0.1, timeout=2)
+    elapsed = time.perf_counter() - start
+
+    # Should take roughly one poll interval to reach completed
+    assert 0.09 <= elapsed <= 0.5
+
+
+@pytest.mark.asyncio
+async def test_async_transport_used_no_threads(monkeypatch):
+    # Make any to_thread usage blow up
+    monkeypatch.setattr(asyncio, "to_thread", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("to_thread not allowed")))
+    # Make sync HttpClient unusable
+    def _boom(*a, **k):
+        raise RuntimeError("sync client should not be used")
+    monkeypatch.setattr(HttpClient, "post", _boom)
+    monkeypatch.setattr(HttpClient, "get", _boom)
+    monkeypatch.setattr(HttpClient, "delete", _boom)
+
+    # Track true async concurrency
+    active = 0
+    max_active = 0
+    async def fake_post(self, endpoint, data, headers=None, timeout=None):
+        nonlocal active, max_active
+        active += 1
+        max_active = max(max_active, active)
+        try:
+            await asyncio.sleep(0.1)
+            return httpx.Response(200, json={"success": True, "data": {}})
+        finally:
+            active -= 1
+
+    monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    await asyncio.gather(
+        client.scrape("https://firecrawl.dev"),
+        client.scrape("https://firecrawl.dev"),
+        client.search("q"),  # uses async search
+    )
+
+    assert max_active >= 2
+