firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/__tests__/unit/v2/methods/aio/test_ensure_async.py

@@ -0,0 +1,117 @@
+import asyncio
+import time
+import httpx
+import pytest
+
+from firecrawl.v2.client_async import AsyncFirecrawlClient
+from firecrawl.v2.utils.http_client_async import AsyncHttpClient
+from firecrawl.v2.utils.http_client import HttpClient
+from firecrawl.v2.methods.aio import batch as aio_batch
+
+
+@pytest.mark.asyncio
+async def test_scrape_concurrency(monkeypatch):
+    async def fake_post(self, endpoint, data, headers=None, timeout=None):
+        await asyncio.sleep(0.1)
+        return httpx.Response(200, json={"success": True, "data": {}})
+
+    monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    start = time.perf_counter()
+    await asyncio.gather(
+        client.scrape("https://firecrawl.dev"),
+        client.scrape("https://firecrawl.dev"),
+        client.scrape("https://firecrawl.dev")
+    )
+    elapsed = time.perf_counter() - start
+
+    # If calls run concurrently, total should be close to single 0.1s delay, not 0.3s
+    assert elapsed < 0.25
+
+
+@pytest.mark.asyncio
+async def test_event_loop_not_blocked(monkeypatch):
+    ticks = 0
+
+    async def ticker():
+        nonlocal ticks
+        for _ in range(5):
+            await asyncio.sleep(0.05)
+            ticks += 1
+
+    async def fake_post(self, endpoint, data, headers=None, timeout=None):
+        await asyncio.sleep(0.2)
+        return httpx.Response(200, json={"success": True, "data": {}})
+
+    monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    await asyncio.gather(ticker(), client.scrape("https://a"))
+    # If scrape awaited properly, ticker should have progressed several steps
+    assert ticks >= 3
+
+
+@pytest.mark.asyncio
+async def test_wait_batch_scrape_polling_interval(monkeypatch):
+    # Simulate one scraping status then completed
+    class S:  # simple status holder
+        def __init__(self, status):
+            self.status = status
+
+    states = ["scraping", "completed"]
+
+    async def fake_status(client, job_id):
+        state = states.pop(0)
+        return S(state)
+
+    monkeypatch.setattr(aio_batch, "get_batch_scrape_status", fake_status)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    start = time.perf_counter()
+    await client.wait_batch_scrape("job-1", poll_interval=0.1, timeout=2)
+    elapsed = time.perf_counter() - start
+
+    # Should take roughly one poll interval to reach completed
+    assert 0.09 <= elapsed <= 0.5
+
+
+@pytest.mark.asyncio
+async def test_async_transport_used_no_threads(monkeypatch):
+    # Make any to_thread usage blow up
+    monkeypatch.setattr(asyncio, "to_thread", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("to_thread not allowed")))
+    # Make sync HttpClient unusable
+    def _boom(*a, **k):
+        raise RuntimeError("sync client should not be used")
+    monkeypatch.setattr(HttpClient, "post", _boom)
+    monkeypatch.setattr(HttpClient, "get", _boom)
+    monkeypatch.setattr(HttpClient, "delete", _boom)
+
+    # Track true async concurrency
+    active = 0
+    max_active = 0
+    async def fake_post(self, endpoint, data, headers=None, timeout=None):
+        nonlocal active, max_active
+        active += 1
+        max_active = max(max_active, active)
+        try:
+            await asyncio.sleep(0.1)
+            return httpx.Response(200, json={"success": True, "data": {}})
+        finally:
+            active -= 1
+
+    monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
+
+    client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
+
+    await asyncio.gather(
+        client.scrape("https://firecrawl.dev"),
+        client.scrape("https://firecrawl.dev"),
+        client.search("q"),  # uses async search
+    )
+
+    assert max_active >= 2
+

firecrawl/__tests__/unit/v2/methods/test_batch_request_preparation.py

@@ -0,0 +1,90 @@
+import pytest
+from firecrawl.v2.types import ScrapeOptions, Location, WebhookConfig
+from firecrawl.v2.methods.batch import prepare_batch_scrape_request
+
+
+class TestBatchScrapeRequestPreparation:
+    """Unit tests for batch scrape request preparation."""
+
+    def test_urls_validation(self):
+        # empty list
+        with pytest.raises(ValueError):
+            prepare_batch_scrape_request([])
+        # invalid protocol
+        with pytest.raises(ValueError):
+            prepare_batch_scrape_request(["example.com"])  # missing http(s)
+        # valid
+        data = prepare_batch_scrape_request(["https://example.com", "http://foo.bar"])
+        assert data["urls"] == ["https://example.com", "http://foo.bar"]
+
+    def test_flatten_scrape_options(self):
+        opts = ScrapeOptions(
+            formats=["markdown", "change_tracking", {"type": "screenshot", "full_page": True, "quality": 80}],
+            include_tags=["main"],
+            exclude_tags=["nav"],
+            only_main_content=True,
+            wait_for=500,
+            timeout=30000,
+            mobile=True,
+            parsers=["pdf"],
+            actions=[{"type": "screenshot", "full_page": True}],
+            location=Location(country="us", languages=["en"]),
+            skip_tls_verification=False,
+            remove_base64_images=False,
+            fast_mode=True,
+            use_mock="test",
+            block_ads=False,
+            proxy="basic",
+            max_age=1000,
+            store_in_cache=False,
+        )
+        data = prepare_batch_scrape_request(["https://example.com"], options=opts)
+
+        # Formats should be at top-level as list, with screenshot normalized to object w/ fullPage
+        assert isinstance(data.get("formats"), list)
+        assert "markdown" in data["formats"]
+        # snake_case format should be converted to camelCase
+        assert "changeTracking" in data["formats"]
+        found_obj = next((f for f in data["formats"] if isinstance(f, dict) and f.get("type") == "screenshot"), None)
+        assert found_obj is not None and found_obj.get("fullPage") is True and found_obj.get("quality") == 80
+
+        # Field conversions to camelCase
+        assert data["includeTags"] == ["main"]
+        assert data["excludeTags"] == ["nav"]
+        assert data["onlyMainContent"] is True
+        assert data["waitFor"] == 500
+        assert data["timeout"] == 30000
+        assert data["mobile"] is True
+        assert data["parsers"] == ["pdf"]
+        assert isinstance(data["actions"], list) and data["actions"][0]["type"] == "screenshot"
+        assert isinstance(data["location"], dict) and data["location"]["country"] == "us"
+        assert data["skipTlsVerification"] is False
+        assert data["removeBase64Images"] is False
+        assert data["fastMode"] is True
+        assert data["useMock"] == "test"
+        assert data["blockAds"] is False
+        assert data["proxy"] == "basic"
+        assert data["maxAge"] == 1000
+        assert data["storeInCache"] is False
+
+    def test_batch_specific_fields(self):
+        webhook = WebhookConfig(url="https://hook.test", headers={"X": "Y"}, events=["completed"]) 
+        data = prepare_batch_scrape_request(
+            ["https://example.com"],
+            webhook=webhook,
+            append_to_id="00000000-0000-0000-0000-000000000000",
+            ignore_invalid_urls=True,
+            max_concurrency=5,
+            zero_data_retention=True,
+            integration="test",
+        )
+        assert isinstance(data["webhook"], dict) and data["webhook"]["url"] == "https://hook.test"
+        assert data["appendToId"] == "00000000-0000-0000-0000-000000000000"
+        assert data["ignoreInvalidURLs"] is True
+        assert data["maxConcurrency"] == 5
+        assert data["zeroDataRetention"] is True
+        assert data["integration"] == "test"
+
+    def test_string_webhook_is_passed_verbatim(self):
+        data = prepare_batch_scrape_request(["https://example.com"], webhook="https://hook.simple")
+        assert data["webhook"] == "https://hook.simple"

firecrawl/__tests__/unit/v2/methods/test_crawl_params.py

@@ -0,0 +1,70 @@
+"""
+Unit tests for crawl params functionality in Firecrawl v2 SDK.
+"""
+
+import pytest
+from firecrawl.v2.types import CrawlParamsRequest, CrawlParamsData
+
+
+class TestCrawlParamsRequest:
+    """Unit tests for CrawlParamsRequest."""
+
+    def test_crawl_params_request_creation(self):
+        """Test creating CrawlParamsRequest with valid data."""
+        request = CrawlParamsRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts"
+        )
+        
+        assert request.url == "https://example.com"
+        assert request.prompt == "Extract all blog posts"
+
+    def test_crawl_params_request_serialization(self):
+        """Test that CrawlParamsRequest serializes correctly."""
+        request = CrawlParamsRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts and documentation"
+        )
+        
+        data = request.model_dump()
+        
+        assert data["url"] == "https://example.com"
+        assert data["prompt"] == "Extract all blog posts and documentation"
+
+
+class TestCrawlParamsData:
+    """Unit tests for CrawlParamsData."""
+
+    def test_crawl_params_data_creation(self):
+        """Test creating CrawlParamsData with minimal data."""
+        data = CrawlParamsData()
+        
+        assert data.include_paths is None
+        assert data.exclude_paths is None
+        assert data.max_discovery_depth is None
+        assert data.ignore_sitemap is False
+        assert data.limit is None
+        assert data.crawl_entire_domain is False
+        assert data.allow_external_links is False
+        assert data.scrape_options is None
+        assert data.warning is None
+
+    def test_crawl_params_data_with_values(self):
+        """Test creating CrawlParamsData with values."""
+        data = CrawlParamsData(
+            include_paths=["/blog/*"],
+            exclude_paths=["/admin/*"],
+            max_discovery_depth=3,
+            limit=50,
+            crawl_entire_domain=True,
+            allow_external_links=False,
+            warning="Test warning"
+        )
+        
+        assert data.include_paths == ["/blog/*"]
+        assert data.exclude_paths == ["/admin/*"]
+        assert data.max_discovery_depth == 3
+        assert data.limit == 50
+        assert data.crawl_entire_domain is True
+        assert data.allow_external_links is False
+        assert data.warning == "Test warning" 

firecrawl/__tests__/unit/v2/methods/test_crawl_request_preparation.py

@@ -0,0 +1,240 @@
+import pytest
+from firecrawl.v2.types import CrawlRequest, ScrapeOptions
+from firecrawl.v2.methods.crawl import _prepare_crawl_request
+
+
+class TestCrawlRequestPreparation:
+    """Unit tests for crawl request preparation."""
+
+    def test_basic_request_preparation(self):
+        """Test basic request preparation with minimal fields."""
+        request = CrawlRequest(url="https://example.com")
+        data = _prepare_crawl_request(request)
+        
+        # Check basic fields
+        assert data["url"] == "https://example.com"
+        
+        # Check that no options are present
+        assert "limit" not in data
+        assert "prompt" not in data
+
+    def test_crawl_options_conversion(self):
+        """Test that CrawlOptions fields are converted to camelCase."""
+        request = CrawlRequest(
+            url="https://example.com",
+            limit=10,
+            max_discovery_depth=3,
+            ignore_sitemap=True,
+            crawl_entire_domain=False,
+            allow_external_links=True
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Check basic field
+        assert data["url"] == "https://example.com"
+        
+        # Check snake_case to camelCase conversions
+        assert "limit" in data
+        assert data["limit"] == 10
+        assert "maxDiscoveryDepth" in data
+        assert data["maxDiscoveryDepth"] == 3
+        assert "ignoreSitemap" in data
+        assert data["ignoreSitemap"] is True
+        assert "crawlEntireDomain" in data
+        assert data["crawlEntireDomain"] is False
+        assert "allowExternalLinks" in data
+        assert data["allowExternalLinks"] is True
+        
+        # Check that snake_case fields are not present
+        assert "ignore_sitemap" not in data
+        assert "crawl_entire_domain" not in data
+        assert "allow_external_links" not in data
+
+    def test_scrape_options_conversion(self):
+        """Test that nested ScrapeOptions are converted to camelCase."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown", "html"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        assert "scrapeOptions" in data
+        assert "scrape_options" not in data
+        
+        # Check nested conversions
+        scrape_data = data["scrapeOptions"]
+        assert "includeTags" in scrape_data
+        assert scrape_data["includeTags"] == ["h1", "h2"]
+        assert "excludeTags" in scrape_data
+        assert scrape_data["excludeTags"] == ["nav"]
+        assert "onlyMainContent" in scrape_data
+        assert scrape_data["onlyMainContent"] is False
+        assert "waitFor" in scrape_data
+        assert scrape_data["waitFor"] == 2000
+        assert "skipTlsVerification" in scrape_data
+        assert scrape_data["skipTlsVerification"] is True
+        assert "removeBase64Images" in scrape_data
+        assert scrape_data["removeBase64Images"] is False
+
+    def test_all_fields_conversion(self):
+        """Test request preparation with all possible fields."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown"],
+            headers={"User-Agent": "Test"},
+            only_main_content=False,
+            mobile=True
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts and documentation",
+            include_paths=["/blog/*", "/docs/*"],
+            exclude_paths=["/admin/*"],
+            max_discovery_depth=3,
+            ignore_sitemap=False,
+            limit=100,
+            crawl_entire_domain=True,
+            allow_external_links=False,
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Check basic fields
+        assert data["url"] == "https://example.com"
+        assert data["prompt"] == "Extract all blog posts and documentation"
+        
+        # Check all CrawlOptions fields
+        assert "includePaths" in data
+        assert data["includePaths"] == ["/blog/*", "/docs/*"]
+        assert "excludePaths" in data
+        assert data["excludePaths"] == ["/admin/*"]
+        assert "maxDiscoveryDepth" in data
+        assert data["maxDiscoveryDepth"] == 3
+        assert "ignoreSitemap" in data
+        assert data["ignoreSitemap"] is False
+        assert "limit" in data
+        assert data["limit"] == 100
+        assert "crawlEntireDomain" in data
+        assert data["crawlEntireDomain"] is True
+        assert "allowExternalLinks" in data
+        assert data["allowExternalLinks"] is False
+        
+        # Check nested scrape options
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        assert "onlyMainContent" in scrape_data
+        assert scrape_data["onlyMainContent"] is False
+        assert "mobile" in scrape_data
+        assert scrape_data["mobile"] is True
+
+    def test_none_values_handling(self):
+        """Test that None values are handled correctly."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt=None,
+            limit=None,
+            scrape_options=None
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Only the required field should be present
+        assert "url" in data
+        assert len(data) == 1  # Only url should be present
+
+    def test_prompt_parameter(self):
+        """Test that prompt parameter is included when provided."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts"
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        assert "url" in data
+        assert "prompt" in data
+        assert data["prompt"] == "Extract all blog posts"
+
+    def test_empty_options(self):
+        """Test that empty options are handled correctly."""
+        request = CrawlRequest(
+            url="https://example.com"
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Should only have the required url field
+        assert "url" in data
+        assert len(data) == 1  # Only url should be present
+
+    def test_validation_integration(self):
+        """Test that validation is called during preparation."""
+        # This should raise an error due to validation
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            request = CrawlRequest(url="")
+            _prepare_crawl_request(request)
+        
+        # This should raise an error due to validation
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            request = CrawlRequest(
+                url="https://example.com",
+                limit=0
+            )
+            _prepare_crawl_request(request)
+
+    def test_scrape_options_shared_function_integration(self):
+        """Test that the shared prepare_scrape_options function is being used."""
+        # Test with all snake_case fields to ensure conversion
+        scrape_opts = ScrapeOptions(
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            wait_for=2000,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Check that scrapeOptions is present and converted
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        
+        # Check all conversions are working
+        assert "includeTags" in scrape_data
+        assert "excludeTags" in scrape_data
+        assert "onlyMainContent" in scrape_data
+        assert "waitFor" in scrape_data
+        assert "skipTlsVerification" in scrape_data
+        assert "removeBase64Images" in scrape_data
+        
+        # Check that snake_case fields are not present
+        assert "include_tags" not in scrape_data
+        assert "exclude_tags" not in scrape_data
+        assert "only_main_content" not in scrape_data
+        assert "wait_for" not in scrape_data
+        assert "skip_tls_verification" not in scrape_data
+        assert "remove_base64_images" not in scrape_data
+        assert "raw_html" not in scrape_data
+        assert "screenshot_full_page" not in scrape_data 

firecrawl/__tests__/unit/v2/methods/test_crawl_validation.py

@@ -0,0 +1,107 @@
+import pytest
+from firecrawl.v2.types import CrawlRequest, ScrapeOptions
+from firecrawl.v2.methods.crawl import _validate_crawl_request
+
+
+class TestCrawlRequestValidation:
+    """Unit tests for crawl request validation."""
+
+    def test_validate_empty_url(self):
+        """Test validation with empty URL."""
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            request = CrawlRequest(url="")
+            _validate_crawl_request(request)
+
+    def test_validate_whitespace_url(self):
+        """Test validation with whitespace-only URL."""
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            request = CrawlRequest(url="   ")
+            _validate_crawl_request(request)
+
+    def test_validate_valid_url(self):
+        """Test validation with valid URL."""
+        request = CrawlRequest(url="https://example.com")
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_invalid_limit(self):
+        """Test validation with invalid limit."""
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            request = CrawlRequest(
+                url="https://example.com",
+                limit=0
+            )
+            _validate_crawl_request(request)
+
+    def test_validate_negative_limit(self):
+        """Test validation with negative limit."""
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            request = CrawlRequest(
+                url="https://example.com",
+                limit=-5
+            )
+            _validate_crawl_request(request)
+
+    def test_validate_valid_limit(self):
+        """Test validation with valid limit."""
+        request = CrawlRequest(
+            url="https://example.com",
+            limit=10
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_with_prompt(self):
+        """Test validation with prompt."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts"
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_with_prompt_and_options(self):
+        """Test validation with prompt and options."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts",
+            limit=10
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_none_options(self):
+        """Test validation with None options."""
+        request = CrawlRequest(url="https://example.com")
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_complex_options(self):
+        """Test validation with complex options."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown"],
+            only_main_content=False,
+            mobile=True
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            limit=50,
+            max_discovery_depth=3,
+            scrape_options=scrape_opts
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_scrape_options_integration(self):
+        """Test that scrape_options validation is integrated."""
+        # Test with valid scrape options
+        scrape_opts = ScrapeOptions(formats=["markdown"], timeout=30000)
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=scrape_opts
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+        # Test with invalid scrape options (should raise error)
+        invalid_scrape_opts = ScrapeOptions(timeout=-1000)
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=invalid_scrape_opts
+        )
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            _validate_crawl_request(request) 

firecrawl/__tests__/unit/v2/methods/test_map_request_preparation.py

@@ -0,0 +1,53 @@
+import pytest
+from firecrawl.v2.types import MapOptions
+from firecrawl.v2.methods.map import _prepare_map_request
+
+
+class TestMapRequestPreparation:
+    """Unit tests for map request preparation."""
+
+    def test_basic_request_preparation(self):
+        data = _prepare_map_request("https://example.com")
+        assert data["url"] == "https://example.com"
+        # Default sitemap handling should be "include" when no flags provided
+        assert "sitemap" not in data  # we only send when options provided
+
+    def test_sitemap_transformations(self):
+        # sitemap -> "only"
+        opts = MapOptions(sitemap="only")
+        data = _prepare_map_request("https://example.com", opts)
+        assert data["sitemap"] == "only"
+
+        # sitemap -> "skip"
+        opts = MapOptions(sitemap="skip")
+        data = _prepare_map_request("https://example.com", opts)
+        assert data["sitemap"] == "skip"
+
+        # default when options present but sitemap left as default -> include
+        opts = MapOptions(search="docs")
+        data = _prepare_map_request("https://example.com", opts)
+        assert data["sitemap"] == "include"
+
+    def test_field_conversions(self):
+        opts = MapOptions(
+            search="docs",
+            include_subdomains=True,
+            limit=25,
+            sitemap="only",
+            timeout=15000,
+        )
+        data = _prepare_map_request("https://example.com", opts)
+
+        assert data["url"] == "https://example.com"
+        assert data["search"] == "docs"
+        assert data["includeSubdomains"] is True
+        assert data["limit"] == 25
+        assert data["sitemap"] == "only"
+        assert data["timeout"] == 15000
+
+    def test_invalid_url(self):
+        with pytest.raises(ValueError):
+            _prepare_map_request("")
+        with pytest.raises(ValueError):
+            _prepare_map_request("   ")
+