firecrawl 4.12.0__py3-none-any.whl

firecrawl/__tests__/unit/v2/methods/test_scrape_request_preparation.py

@@ -0,0 +1,109 @@
+import pytest
+from firecrawl.v2.types import ScrapeOptions, Viewport, ScreenshotAction
+from firecrawl.v2.methods.scrape import _prepare_scrape_request
+
+
+class TestScrapeRequestPreparation:
+    """Unit tests for scrape request preparation."""
+
+    def test_basic_request_preparation(self):
+        """Test basic request preparation with minimal fields."""
+        data = _prepare_scrape_request("https://example.com")
+        
+        # Check basic fields
+        assert data["url"] == "https://example.com"
+        
+        # Check that no options are present
+        assert "formats" not in data
+        assert "headers" not in data
+
+    def test_scrape_options_conversion(self):
+        """Test that ScrapeOptions fields are converted to camelCase."""
+        options = ScrapeOptions(
+            formats=["markdown", "html"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        
+        data = _prepare_scrape_request("https://example.com", options)
+        
+        # Check basic field
+        assert data["url"] == "https://example.com"
+        
+        # Check snake_case to camelCase conversions
+        assert "includeTags" in data
+        assert data["includeTags"] == ["h1", "h2"]
+        assert "excludeTags" in data
+        assert data["excludeTags"] == ["nav"]
+        assert "onlyMainContent" in data
+        assert data["onlyMainContent"] is False
+        assert "waitFor" in data
+        assert data["waitFor"] == 2000
+        assert "skipTlsVerification" in data
+        assert data["skipTlsVerification"] is True
+        assert "removeBase64Images" in data
+        assert data["removeBase64Images"] is False
+        
+        # Check that snake_case fields are not present
+        assert "include_tags" not in data
+        assert "exclude_tags" not in data
+        assert "only_main_content" not in data
+
+    def test_actions_conversion(self):
+        """Test that actions are converted to camelCase."""
+        viewport = Viewport(width=800, height=600)
+        action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)
+        
+        options = ScrapeOptions(actions=[action])
+        data = _prepare_scrape_request("https://example.com", options)
+        
+        assert "actions" in data
+        assert len(data["actions"]) == 1
+        
+        action_data = data["actions"][0]
+        assert action_data["type"] == "screenshot"
+        assert action_data["fullPage"] is False
+        assert action_data["quality"] == 80
+        assert "viewport" in action_data
+
+    def test_none_options_handling(self):
+        """Test handling of None options."""
+        data = _prepare_scrape_request("https://example.com", None)
+        
+        assert data["url"] == "https://example.com"
+        # Should not have any option fields
+        assert len(data) == 1
+
+    def test_empty_url_validation(self):
+        """Test validation with empty URL."""
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            _prepare_scrape_request("")
+
+    def test_whitespace_url_validation(self):
+        """Test validation with whitespace-only URL."""
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            _prepare_scrape_request("   ")
+
+    def test_all_params_including_integration(self):
+        opts = ScrapeOptions(
+            formats=["markdown"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False,
+            integration="  _unit-test  ",
+        )
+        data = _prepare_scrape_request("https://example.com", opts)
+        assert data["integration"] == "_unit-test"

firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py

@@ -0,0 +1,169 @@
+import pytest
+from firecrawl.v2.types import SearchRequest, ScrapeOptions, Source
+from firecrawl.v2.methods.search import _prepare_search_request
+
+
+class TestSearchRequestPreparation:
+    """Unit tests for search request preparation."""
+
+    def test_basic_request_preparation(self):
+        """Test basic request preparation with minimal fields."""
+        request = SearchRequest(query="test query")
+        data = _prepare_search_request(request)
+    
+        # Check basic fields
+        assert data["query"] == "test query"
+        assert data["limit"] == 5
+        assert data["timeout"] == 300000
+        
+        # Check that snake_case fields are not present
+        assert "ignore_invalid_urls" not in data
+        assert "scrape_options" not in data
+
+    def test_all_fields_conversion(self):
+        """Test request preparation with all possible fields."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        
+        request = SearchRequest(
+            query="test query",
+            sources=["web", "news"],
+            limit=10,
+            tbs="qdr:w",
+            location="US",
+            ignore_invalid_urls=False,
+            timeout=30000,
+            scrape_options=scrape_opts,
+            integration="  _e2e-test  ",
+        )
+        
+        data = _prepare_search_request(request)
+        
+        # Check all basic fields
+        assert data["query"] == "test query"
+        assert data["limit"] == 10
+        assert data["tbs"] == "qdr:w"
+        assert data["location"] == "US"
+        assert data["timeout"] == 30000
+        
+        # Check snake_case to camelCase conversions
+        assert "ignoreInvalidURLs" in data
+        assert data["ignoreInvalidURLs"] is False
+        assert "ignore_invalid_urls" not in data
+        
+        assert "scrapeOptions" in data
+        assert "scrape_options" not in data
+        
+        # Check sources
+        assert "sources" in data
+        assert len(data["sources"]) == 2
+        assert data["sources"][0]["type"] == "web"
+        assert data["sources"][1]["type"] == "news"
+        
+        # Check nested scrape options conversions
+        scrape_data = data["scrapeOptions"]
+        assert "includeTags" in scrape_data
+        assert scrape_data["includeTags"] == ["h1", "h2"]
+        assert "excludeTags" in scrape_data
+        assert scrape_data["excludeTags"] == ["nav"]
+        assert "onlyMainContent" in scrape_data
+        assert scrape_data["onlyMainContent"] is False
+        assert "waitFor" in scrape_data
+        assert scrape_data["waitFor"] == 2000
+        assert "skipTlsVerification" in scrape_data
+        assert scrape_data["skipTlsVerification"] is True
+        assert "removeBase64Images" in scrape_data
+        assert scrape_data["removeBase64Images"] is False
+        assert data["integration"] == "_e2e-test"
+
+    def test_exclude_none_behavior(self):
+        """Test that exclude_none=True behavior is working."""
+        request = SearchRequest(
+            query="test",
+            sources=None,
+            limit=None,
+            tbs=None,
+            location=None,
+            ignore_invalid_urls=None,
+            timeout=None,
+            scrape_options=None
+        )
+    
+        data = _prepare_search_request(request)
+    
+        # When limit and timeout are explicitly None, they should be excluded
+        assert "query" in data
+        assert len(data) == 1  # Only query should be present
+
+    def test_empty_scrape_options(self):
+        """Test that empty scrape options are handled correctly."""
+        scrape_opts = ScrapeOptions()  # All defaults
+        
+        request = SearchRequest(
+            query="test",
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_search_request(request)
+        
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        
+        # Should have default values
+        assert "onlyMainContent" in scrape_data
+        assert scrape_data["onlyMainContent"] is True
+        assert "mobile" in scrape_data
+        assert scrape_data["mobile"] is False
+
+    def test_scrape_options_shared_function_integration(self):
+        """Test that the shared prepare_scrape_options function is being used."""
+        # Test with all snake_case fields to ensure conversion
+        scrape_opts = ScrapeOptions(
+            formats=["markdown", "rawHtml"],
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            wait_for=2000,
+            skip_tls_verification=True,
+            remove_base64_images=False
+            # Note: raw_html should be in formats array, not as a separate field
+        )
+        
+        request = SearchRequest(
+            query="test",
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_search_request(request)
+        
+        # Check that scrapeOptions is present and converted
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        
+        # Check all conversions are working
+        assert "formats" in scrape_data
+        assert scrape_data["formats"] == ["markdown", "rawHtml"]
+        assert "includeTags" in scrape_data
+        assert "excludeTags" in scrape_data
+        assert "onlyMainContent" in scrape_data
+        assert "waitFor" in scrape_data
+        assert "skipTlsVerification" in scrape_data
+        assert "removeBase64Images" in scrape_data
+        
+        # Check that snake_case fields are not present
+        assert "include_tags" not in scrape_data
+        assert "exclude_tags" not in scrape_data
+        assert "only_main_content" not in scrape_data
+        assert "wait_for" not in scrape_data
+        assert "skip_tls_verification" not in scrape_data
+        assert "remove_base64_images" not in scrape_data

firecrawl/__tests__/unit/v2/methods/test_search_validation.py

@@ -0,0 +1,236 @@
+import pytest
+from firecrawl.v2.types import SearchRequest, Source, ScrapeOptions, ScrapeFormats
+from firecrawl.v2.methods.search import _validate_search_request
+
+
+class TestSearchValidation:
+    """Unit tests for search request validation."""
+
+    def test_validate_empty_query(self):
+        """Test validation of empty query."""
+        request = SearchRequest(query="")
+        with pytest.raises(ValueError, match="Query cannot be empty"):
+            _validate_search_request(request)
+
+        request = SearchRequest(query="   ")
+        with pytest.raises(ValueError, match="Query cannot be empty"):
+            _validate_search_request(request)
+
+    def test_validate_invalid_limit(self):
+        """Test validation of invalid limits."""
+        # Zero limit
+        request = SearchRequest(query="test", limit=0)
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            _validate_search_request(request)
+
+        # Negative limit
+        request = SearchRequest(query="test", limit=-1)
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            _validate_search_request(request)
+
+        # Too high limit
+        request = SearchRequest(query="test", limit=101)
+        with pytest.raises(ValueError, match="Limit cannot exceed 100"):
+            _validate_search_request(request)
+
+    def test_validate_invalid_timeout(self):
+        """Test validation of invalid timeouts."""
+        # Zero timeout
+        request = SearchRequest(query="test", timeout=0)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            _validate_search_request(request)
+
+        # Negative timeout
+        request = SearchRequest(query="test", timeout=-1000)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            _validate_search_request(request)
+
+        # Too high timeout
+        request = SearchRequest(query="test", timeout=300001)
+        with pytest.raises(ValueError, match="Timeout cannot exceed 300000ms"):
+            _validate_search_request(request)
+
+    def test_validate_invalid_sources(self):
+        """Test validation of invalid sources."""
+        # Invalid string source
+        request = SearchRequest(query="test", sources=["invalid_source"])
+        with pytest.raises(ValueError, match="Invalid source type"):
+            _validate_search_request(request)
+
+        # Invalid object source
+        request = SearchRequest(query="test", sources=[Source(type="invalid_source")])
+        with pytest.raises(ValueError, match="Invalid source type"):
+            _validate_search_request(request)
+
+        # Mixed valid/invalid sources
+        request = SearchRequest(query="test", sources=["web", "invalid_source"])
+        with pytest.raises(ValueError, match="Invalid source type"):
+            _validate_search_request(request)
+
+    def test_validate_invalid_location(self):
+        """Test validation of invalid location."""
+        # Empty location
+        request = SearchRequest(query="test", location="")
+        with pytest.raises(ValueError, match="Location must be a non-empty string"):
+            _validate_search_request(request)
+
+        # Whitespace location
+        request = SearchRequest(query="test", location="   ")
+        with pytest.raises(ValueError, match="Location must be a non-empty string"):
+            _validate_search_request(request)
+
+    def test_validate_invalid_tbs(self):
+        """Test validation of invalid tbs values."""
+        invalid_tbs_values = ["invalid", "qdr:x", "yesterday", "last_week"]
+
+        for invalid_tbs in invalid_tbs_values:
+            request = SearchRequest(query="test", tbs=invalid_tbs)
+            with pytest.raises(ValueError, match="Invalid tbs value"):
+                _validate_search_request(request)
+
+    def test_validate_custom_date_ranges(self):
+        """Test validation of custom date range formats."""
+        valid_custom_ranges = [
+            "cdr:1,cd_min:1/1/2024,cd_max:12/31/2024",
+            "cdr:1,cd_min:12/1/2024,cd_max:12/31/2024",
+            "cdr:1,cd_min:2/28/2023,cd_max:3/1/2023",
+            "cdr:1,cd_min:10/15/2023,cd_max:11/15/2023"
+        ]
+
+        for valid_range in valid_custom_ranges:
+            request = SearchRequest(query="test", tbs=valid_range)
+            validated = _validate_search_request(request)
+            assert validated == request
+
+    def test_validate_invalid_custom_date_ranges(self):
+        """Test validation of invalid custom date range formats."""
+        # Invalid custom date ranges
+        invalid_custom_ranges = [
+            "cdr:1,cd_min:2/28/2023",  # Missing cd_max
+            "cdr:1,cd_max:2/28/2023",  # Missing cd_min
+            "cdr:2,cd_min:1/1/2024,cd_max:12/31/2024",  # Wrong cdr value
+            "cdr:cd_min:1/1/2024,cd_max:12/31/2024",  # Missing :1
+            "custom:1,cd_min:1/1/2024,cd_max:12/31/2024"  # Wrong prefix
+        ]
+
+        for invalid_range in invalid_custom_ranges:
+            request = SearchRequest(query="test", tbs=invalid_range)
+            with pytest.raises(ValueError, match="Invalid"):
+                _validate_search_request(request)
+
+    def test_validate_valid_requests(self):
+        """Test that valid requests pass validation."""
+        # Minimal valid request
+        request = SearchRequest(query="test")
+        validated = _validate_search_request(request)
+        assert validated == request
+
+        # Request with all optional parameters
+        request = SearchRequest(
+            query="test query",
+            sources=["web", "news"],
+            limit=10,
+            tbs="qdr:w",
+            location="US",
+            ignore_invalid_urls=False,
+            timeout=30000
+        )
+        validated = _validate_search_request(request)
+        assert validated == request
+
+        # Request with object sources
+        request = SearchRequest(
+            query="test",
+            sources=[Source(type="web"), Source(type="images")]
+        )
+        validated = _validate_search_request(request)
+        assert validated == request
+
+    def test_validate_edge_cases(self):
+        """Test edge cases and boundary values."""
+        # Maximum valid limit
+        request = SearchRequest(query="test", limit=100)
+        validated = _validate_search_request(request)
+        assert validated == request
+
+        # Maximum valid timeout
+        request = SearchRequest(query="test", timeout=300000)
+        validated = _validate_search_request(request)
+        assert validated == request
+
+        # Minimum valid limit
+        request = SearchRequest(query="test", limit=1)
+        validated = _validate_search_request(request)
+        assert validated == request
+
+        # Minimum valid timeout
+        request = SearchRequest(query="test", timeout=1)
+        validated = _validate_search_request(request)
+        assert validated == request
+
+    def test_validate_none_values(self):
+        """Test that None values for optional fields are handled correctly."""
+        request = SearchRequest(
+            query="test",
+            sources=None,
+            limit=None,
+            tbs=None,
+            location=None,
+            ignore_invalid_urls=None,
+            timeout=None
+        )
+        validated = _validate_search_request(request)
+        assert validated == request
+
+    def test_validate_scrape_options_integration(self):
+        """Test that scrape_options validation is integrated."""
+        # Test with valid scrape options
+        scrape_opts = ScrapeOptions(formats=["markdown"], timeout=30000)
+        request = SearchRequest(query="test", scrape_options=scrape_opts)
+        validated = _validate_search_request(request)
+        assert validated == request
+
+        # Test with invalid scrape options (should raise error)
+        invalid_scrape_opts = ScrapeOptions(timeout=-1000)
+        request = SearchRequest(query="test", scrape_options=invalid_scrape_opts)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            _validate_search_request(request)
+
+
+
+
+
+class TestSearchRequestModel:
+    """Unit tests for SearchRequest model behavior."""
+
+    def test_default_values(self):
+        """Test that default values are set correctly."""
+        request = SearchRequest(query="test")
+        assert request.limit == 5
+        assert request.ignore_invalid_urls is None  # No default in model
+        assert request.timeout == 300000
+        assert request.sources is None
+        assert request.tbs is None
+        assert request.location is None
+        assert request.scrape_options is None
+
+    def test_field_aliases(self):
+        """Test that field aliases work correctly for API serialization."""
+        # Test with None value (no default)
+        request1 = SearchRequest(query="test")
+        data1 = request1.model_dump(by_alias=True)
+        assert "ignore_invalid_urls" in data1  # No alias, uses snake_case
+        assert data1["ignore_invalid_urls"] is None
+
+        # Test with explicit False value
+        request2 = SearchRequest(
+            query="test",
+            ignore_invalid_urls=False,
+            scrape_options=ScrapeOptions(formats=["markdown"])
+        )
+
+        # Check that aliases are used in model_dump with by_alias=True
+        data2 = request2.model_dump(by_alias=True)
+        assert "ignore_invalid_urls" in data2  # No alias, uses snake_case
+        assert "scrape_options" in data2  # No alias, uses snake_case
+        assert data2["ignore_invalid_urls"] is False

firecrawl/__tests__/unit/v2/methods/test_usage_types.py

@@ -0,0 +1,18 @@
+from firecrawl.v2.types import ConcurrencyCheck, CreditUsage, TokenUsage
+
+
+class TestUsageTypes:
+    def test_concurrency_check_model(self):
+        cc = ConcurrencyCheck(concurrency=3, max_concurrency=10)
+        assert cc.concurrency == 3
+        assert cc.max_concurrency == 10
+
+    def test_credit_usage_model(self):
+        cu = CreditUsage(remaining_credits=123)
+        assert isinstance(cu.remaining_credits, int)
+        assert cu.remaining_credits == 123
+
+    def test_token_usage_model(self):
+        tu = TokenUsage(remaining_tokens=10)
+        assert tu.remaining_tokens == 10
+

firecrawl/__tests__/unit/v2/methods/test_webhook.py

@@ -0,0 +1,123 @@
+"""
+Unit tests for webhook functionality in Firecrawl v2 SDK.
+"""
+
+import pytest
+from firecrawl.v2.types import WebhookConfig, CrawlRequest
+from firecrawl.v2.methods.crawl import _prepare_crawl_request
+
+class TestWebhookConfig:
+    """Test WebhookConfig class functionality."""
+    
+    def test_webhook_config_creation_minimal(self):
+        """Test creating WebhookConfig with minimal parameters."""
+        webhook = WebhookConfig(url="https://example.com/webhook")
+        assert webhook.url == "https://example.com/webhook"
+        assert webhook.headers is None
+        assert webhook.metadata is None
+        assert webhook.events is None
+    
+    def test_webhook_config_creation_full(self):
+        """Test creating WebhookConfig with all parameters."""
+        webhook = WebhookConfig(
+            url="https://example.com/webhook",
+            headers={"Authorization": "Bearer token"},
+            metadata={"project": "test"},
+            events=["completed", "failed"]
+        )
+        assert webhook.url == "https://example.com/webhook"
+        assert webhook.headers == {"Authorization": "Bearer token"}
+        assert webhook.metadata == {"project": "test"}
+        assert webhook.events == ["completed", "failed"]
+    
+    def test_webhook_config_validation(self):
+        """Test WebhookConfig validation."""
+        # URL is required
+        with pytest.raises(Exception):  # Pydantic validation error
+            WebhookConfig()
+
+
+class TestCrawlRequestWebhook:
+    """Test CrawlRequest webhook functionality."""
+    
+    def test_crawl_request_with_string_webhook(self):
+        """Test CrawlRequest with string webhook."""
+        request = CrawlRequest(
+            url="https://example.com",
+            webhook="https://example.com/webhook"
+        )
+        
+        data = _prepare_crawl_request(request)
+        assert data["webhook"] == "https://example.com/webhook"
+    
+    def test_crawl_request_with_webhook_config(self):
+        """Test CrawlRequest with WebhookConfig object."""
+        webhook_config = WebhookConfig(
+            url="https://example.com/webhook",
+            headers={"Authorization": "Bearer token"},
+            events=["completed"]
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            webhook=webhook_config
+        )
+        
+        data = _prepare_crawl_request(request)
+        assert data["webhook"]["url"] == "https://example.com/webhook"
+        assert data["webhook"]["headers"] == {"Authorization": "Bearer token"}
+        assert data["webhook"]["events"] == ["completed"]
+    
+    def test_crawl_request_without_webhook(self):
+        """Test CrawlRequest without webhook."""
+        request = CrawlRequest(url="https://example.com")
+        
+        data = _prepare_crawl_request(request)
+        assert "webhook" not in data
+    
+    def test_crawl_request_webhook_serialization(self):
+        """Test that webhook config is properly serialized."""
+        webhook_config = WebhookConfig(
+            url="https://example.com/webhook",
+            headers={"Content-Type": "application/json"},
+            metadata={"test": "value"},
+            events=["page", "completed"]
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            webhook=webhook_config
+        )
+        
+        data = _prepare_crawl_request(request)
+        webhook_data = data["webhook"]
+        
+        # Check that all fields are properly serialized
+        assert webhook_data["url"] == "https://example.com/webhook"
+        assert webhook_data["headers"] == {"Content-Type": "application/json"}
+        assert webhook_data["metadata"] == {"test": "value"}
+        assert webhook_data["events"] == ["page", "completed"]
+    
+    def test_crawl_request_webhook_with_none_values(self):
+        """Test webhook config with None values are excluded from serialization."""
+        webhook_config = WebhookConfig(
+            url="https://example.com/webhook",
+            headers=None,
+            metadata=None,
+            events=None
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            webhook=webhook_config
+        )
+        
+        data = _prepare_crawl_request(request)
+        webhook_data = data["webhook"]
+        
+        # Only url should be present, None values should be excluded
+        assert webhook_data["url"] == "https://example.com/webhook"
+        assert "headers" not in webhook_data
+        assert "metadata" not in webhook_data
+        assert "events" not in webhook_data 
+