firecrawl 4.12.0__py3-none-any.whl

firecrawl/__tests__/unit/v2/methods/test_branding.py

@@ -0,0 +1,214 @@
+import pytest
+from unittest.mock import Mock, MagicMock
+from firecrawl.v2.methods.scrape import scrape
+from firecrawl.v2.types import ScrapeOptions, Document
+
+
+class TestBrandingFormat:
+    """Unit tests for branding format support."""
+
+    def test_scrape_with_branding_format_returns_branding_data(self):
+        """Test that scraping with branding format returns branding data."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "data": {
+                "markdown": "# Example",
+                "branding": {
+                    "colorScheme": "light",
+                    "colors": {
+                        "primary": "#E11D48",
+                        "secondary": "#3B82F6",
+                        "accent": "#F59E0B"
+                    },
+                    "typography": {
+                        "fontFamilies": {
+                            "primary": "Inter",
+                            "heading": "Poppins"
+                        },
+                        "fontSizes": {
+                            "h1": "2.5rem",
+                            "body": "1rem"
+                        }
+                    },
+                    "spacing": {
+                        "baseUnit": 8
+                    },
+                    "components": {
+                        "buttonPrimary": {
+                            "background": "#E11D48",
+                            "textColor": "#FFFFFF",
+                            "borderRadius": "0.5rem"
+                        }
+                    }
+                }
+            }
+        }
+
+        mock_client = Mock()
+        mock_client.post.return_value = mock_response
+
+        result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))
+
+        assert result.branding is not None
+        assert result.branding.color_scheme == "light"
+        assert result.branding.colors["primary"] == "#E11D48"
+        assert result.branding.typography["fontFamilies"]["primary"] == "Inter"
+        assert result.branding.spacing["baseUnit"] == 8
+        assert result.branding.components["buttonPrimary"]["background"] == "#E11D48"
+
+    def test_scrape_with_branding_and_markdown_formats_returns_both(self):
+        """Test that scraping with both branding and markdown formats returns both."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "data": {
+                "markdown": "# Example Content",
+                "branding": {
+                    "colorScheme": "dark",
+                    "colors": {
+                        "primary": "#10B981"
+                    },
+                    "typography": {
+                        "fontFamilies": {
+                            "primary": "Roboto"
+                        }
+                    }
+                }
+            }
+        }
+
+        mock_client = Mock()
+        mock_client.post.return_value = mock_response
+
+        result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown", "branding"]))
+
+        assert result.markdown == "# Example Content"
+        assert result.branding is not None
+        assert result.branding.color_scheme == "dark"
+        assert result.branding.colors["primary"] == "#10B981"
+
+    def test_scrape_without_branding_format_does_not_return_branding(self):
+        """Test that scraping without branding format does not return branding."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "data": {
+                "markdown": "# Example"
+            }
+        }
+
+        mock_client = Mock()
+        mock_client.post.return_value = mock_response
+
+        result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown"]))
+
+        assert result.markdown == "# Example"
+        assert result.branding is None
+
+    def test_branding_format_with_all_nested_fields(self):
+        """Test branding format with all nested fields populated."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "data": {
+                "branding": {
+                    "colorScheme": "light",
+                    "logo": "https://example.com/logo.png",
+                    "fonts": [
+                        {"family": "Inter", "weight": 400},
+                        {"family": "Poppins", "weight": 700}
+                    ],
+                    "colors": {
+                        "primary": "#E11D48",
+                        "background": "#FFFFFF"
+                    },
+                    "typography": {
+                        "fontFamilies": {"primary": "Inter"},
+                        "fontStacks": {"body": ["Inter", "sans-serif"]},
+                        "fontSizes": {"h1": "2.5rem"},
+                        "lineHeights": {"body": 1.5},
+                        "fontWeights": {"regular": 400}
+                    },
+                    "spacing": {
+                        "baseUnit": 8,
+                        "padding": {"sm": 8, "md": 16}
+                    },
+                    "components": {
+                        "buttonPrimary": {
+                            "background": "#E11D48",
+                            "textColor": "#FFFFFF"
+                        }
+                    },
+                    "icons": {
+                        "style": "outline",
+                        "primaryColor": "#E11D48"
+                    },
+                    "images": {
+                        "logo": "https://example.com/logo.png",
+                        "favicon": "https://example.com/favicon.ico"
+                    },
+                    "animations": {
+                        "transitionDuration": "200ms",
+                        "easing": "ease-in-out"
+                    },
+                    "layout": {
+                        "grid": {"columns": 12, "maxWidth": "1200px"},
+                        "headerHeight": "64px"
+                    },
+                    "tone": {
+                        "voice": "professional",
+                        "emojiUsage": "minimal"
+                    },
+                    "personality": {
+                        "tone": "professional",
+                        "energy": "medium",
+                        "targetAudience": "developers"
+                    }
+                }
+            }
+        }
+
+        mock_client = Mock()
+        mock_client.post.return_value = mock_response
+
+        result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))
+
+        assert result.branding is not None
+        assert result.branding.color_scheme == "light"
+        assert result.branding.logo == "https://example.com/logo.png"
+        assert len(result.branding.fonts) == 2
+        assert result.branding.typography["fontStacks"]["body"] == ["Inter", "sans-serif"]
+        assert result.branding.spacing["padding"] == {"sm": 8, "md": 16}
+        assert result.branding.icons["style"] == "outline"
+        assert result.branding.images["favicon"] == "https://example.com/favicon.ico"
+        assert result.branding.animations["easing"] == "ease-in-out"
+        assert result.branding.layout["grid"]["columns"] == 12
+        assert result.branding.personality["tone"] == "professional"
+
+    def test_branding_colorscheme_normalization(self):
+        """Test that colorScheme is normalized to color_scheme."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "data": {
+                "branding": {
+                    "colorScheme": "dark",
+                    "colors": {"primary": "#000000"}
+                }
+            }
+        }
+
+        mock_client = Mock()
+        mock_client.post.return_value = mock_response
+
+        result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))
+
+        assert result.branding is not None
+        assert result.branding.color_scheme == "dark"
+        assert not hasattr(result.branding, "colorScheme")

firecrawl/__tests__/unit/v2/methods/test_crawl_params.py

@@ -0,0 +1,70 @@
+"""
+Unit tests for crawl params functionality in Firecrawl v2 SDK.
+"""
+
+import pytest
+from firecrawl.v2.types import CrawlParamsRequest, CrawlParamsData
+
+
+class TestCrawlParamsRequest:
+    """Unit tests for CrawlParamsRequest."""
+
+    def test_crawl_params_request_creation(self):
+        """Test creating CrawlParamsRequest with valid data."""
+        request = CrawlParamsRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts"
+        )
+        
+        assert request.url == "https://example.com"
+        assert request.prompt == "Extract all blog posts"
+
+    def test_crawl_params_request_serialization(self):
+        """Test that CrawlParamsRequest serializes correctly."""
+        request = CrawlParamsRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts and documentation"
+        )
+        
+        data = request.model_dump()
+        
+        assert data["url"] == "https://example.com"
+        assert data["prompt"] == "Extract all blog posts and documentation"
+
+
+class TestCrawlParamsData:
+    """Unit tests for CrawlParamsData."""
+
+    def test_crawl_params_data_creation(self):
+        """Test creating CrawlParamsData with minimal data."""
+        data = CrawlParamsData()
+        
+        assert data.include_paths is None
+        assert data.exclude_paths is None
+        assert data.max_discovery_depth is None
+        assert data.ignore_sitemap is False
+        assert data.limit is None
+        assert data.crawl_entire_domain is False
+        assert data.allow_external_links is False
+        assert data.scrape_options is None
+        assert data.warning is None
+
+    def test_crawl_params_data_with_values(self):
+        """Test creating CrawlParamsData with values."""
+        data = CrawlParamsData(
+            include_paths=["/blog/*"],
+            exclude_paths=["/admin/*"],
+            max_discovery_depth=3,
+            limit=50,
+            crawl_entire_domain=True,
+            allow_external_links=False,
+            warning="Test warning"
+        )
+        
+        assert data.include_paths == ["/blog/*"]
+        assert data.exclude_paths == ["/admin/*"]
+        assert data.max_discovery_depth == 3
+        assert data.limit == 50
+        assert data.crawl_entire_domain is True
+        assert data.allow_external_links is False
+        assert data.warning == "Test warning" 

firecrawl/__tests__/unit/v2/methods/test_crawl_request_preparation.py

@@ -0,0 +1,240 @@
+import pytest
+from firecrawl.v2.types import CrawlRequest, ScrapeOptions
+from firecrawl.v2.methods.crawl import _prepare_crawl_request
+
+
+class TestCrawlRequestPreparation:
+    """Unit tests for crawl request preparation."""
+
+    def test_basic_request_preparation(self):
+        """Test basic request preparation with minimal fields."""
+        request = CrawlRequest(url="https://example.com")
+        data = _prepare_crawl_request(request)
+        
+        # Check basic fields
+        assert data["url"] == "https://example.com"
+        
+        # Check that no options are present
+        assert "limit" not in data
+        assert "prompt" not in data
+
+    def test_crawl_options_conversion(self):
+        """Test that CrawlOptions fields are converted to camelCase."""
+        request = CrawlRequest(
+            url="https://example.com",
+            limit=10,
+            max_discovery_depth=3,
+            sitemap="skip",
+            crawl_entire_domain=False,
+            allow_external_links=True
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Check basic field
+        assert data["url"] == "https://example.com"
+        
+        # Check snake_case to camelCase conversions
+        assert "limit" in data
+        assert data["limit"] == 10
+        assert "maxDiscoveryDepth" in data
+        assert data["maxDiscoveryDepth"] == 3
+        assert "sitemap" in data
+        assert data["sitemap"] == "skip"
+        assert "crawlEntireDomain" in data
+        assert data["crawlEntireDomain"] is False
+        assert "allowExternalLinks" in data
+        assert data["allowExternalLinks"] is True
+        
+        # Check that snake_case fields are not present
+        assert "ignore_sitemap" not in data
+        assert "crawl_entire_domain" not in data
+        assert "allow_external_links" not in data
+
+    def test_scrape_options_conversion(self):
+        """Test that nested ScrapeOptions are converted to camelCase."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown", "html"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        assert "scrapeOptions" in data
+        assert "scrape_options" not in data
+        
+        # Check nested conversions
+        scrape_data = data["scrapeOptions"]
+        assert "includeTags" in scrape_data
+        assert scrape_data["includeTags"] == ["h1", "h2"]
+        assert "excludeTags" in scrape_data
+        assert scrape_data["excludeTags"] == ["nav"]
+        assert "onlyMainContent" in scrape_data
+        assert scrape_data["onlyMainContent"] is False
+        assert "waitFor" in scrape_data
+        assert scrape_data["waitFor"] == 2000
+        assert "skipTlsVerification" in scrape_data
+        assert scrape_data["skipTlsVerification"] is True
+        assert "removeBase64Images" in scrape_data
+        assert scrape_data["removeBase64Images"] is False
+
+    def test_all_fields_conversion(self):
+        """Test request preparation with all possible fields."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown"],
+            headers={"User-Agent": "Test"},
+            only_main_content=False,
+            mobile=True
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts and documentation",
+            include_paths=["/blog/*", "/docs/*"],
+            exclude_paths=["/admin/*"],
+            max_discovery_depth=3,
+            sitemap="include",
+            limit=100,
+            crawl_entire_domain=True,
+            allow_external_links=False,
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Check basic fields
+        assert data["url"] == "https://example.com"
+        assert data["prompt"] == "Extract all blog posts and documentation"
+        
+        # Check all CrawlOptions fields
+        assert "includePaths" in data
+        assert data["includePaths"] == ["/blog/*", "/docs/*"]
+        assert "excludePaths" in data
+        assert data["excludePaths"] == ["/admin/*"]
+        assert "maxDiscoveryDepth" in data
+        assert data["maxDiscoveryDepth"] == 3
+        assert "sitemap" in data
+        assert data["sitemap"] == "include"
+        assert "limit" in data
+        assert data["limit"] == 100
+        assert "crawlEntireDomain" in data
+        assert data["crawlEntireDomain"] is True
+        assert "allowExternalLinks" in data
+        assert data["allowExternalLinks"] is False
+        
+        # Check nested scrape options
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        assert "onlyMainContent" in scrape_data
+        assert scrape_data["onlyMainContent"] is False
+        assert "mobile" in scrape_data
+        assert scrape_data["mobile"] is True
+
+    def test_none_values_handling(self):
+        """Test that None values are handled correctly."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt=None,
+            limit=None,
+            scrape_options=None
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Only the required field should be present
+        assert "url" in data
+        assert len(data) == 1  # Only url should be present
+
+    def test_prompt_parameter(self):
+        """Test that prompt parameter is included when provided."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts"
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        assert "url" in data
+        assert "prompt" in data
+        assert data["prompt"] == "Extract all blog posts"
+
+    def test_empty_options(self):
+        """Test that empty options are handled correctly."""
+        request = CrawlRequest(
+            url="https://example.com"
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Should only have the required url field
+        assert "url" in data
+        assert len(data) == 1  # Only url should be present
+
+    def test_validation_integration(self):
+        """Test that validation is called during preparation."""
+        # This should raise an error due to validation
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            request = CrawlRequest(url="")
+            _prepare_crawl_request(request)
+        
+        # This should raise an error due to validation
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            request = CrawlRequest(
+                url="https://example.com",
+                limit=0
+            )
+            _prepare_crawl_request(request)
+
+    def test_scrape_options_shared_function_integration(self):
+        """Test that the shared prepare_scrape_options function is being used."""
+        # Test with all snake_case fields to ensure conversion
+        scrape_opts = ScrapeOptions(
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            wait_for=2000,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=scrape_opts
+        )
+        
+        data = _prepare_crawl_request(request)
+        
+        # Check that scrapeOptions is present and converted
+        assert "scrapeOptions" in data
+        scrape_data = data["scrapeOptions"]
+        
+        # Check all conversions are working
+        assert "includeTags" in scrape_data
+        assert "excludeTags" in scrape_data
+        assert "onlyMainContent" in scrape_data
+        assert "waitFor" in scrape_data
+        assert "skipTlsVerification" in scrape_data
+        assert "removeBase64Images" in scrape_data
+        
+        # Check that snake_case fields are not present
+        assert "include_tags" not in scrape_data
+        assert "exclude_tags" not in scrape_data
+        assert "only_main_content" not in scrape_data
+        assert "wait_for" not in scrape_data
+        assert "skip_tls_verification" not in scrape_data
+        assert "remove_base64_images" not in scrape_data
+        assert "raw_html" not in scrape_data
+        assert "screenshot_full_page" not in scrape_data 

firecrawl/__tests__/unit/v2/methods/test_crawl_validation.py

@@ -0,0 +1,107 @@
+import pytest
+from firecrawl.v2.types import CrawlRequest, ScrapeOptions
+from firecrawl.v2.methods.crawl import _validate_crawl_request
+
+
+class TestCrawlRequestValidation:
+    """Unit tests for crawl request validation."""
+
+    def test_validate_empty_url(self):
+        """Test validation with empty URL."""
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            request = CrawlRequest(url="")
+            _validate_crawl_request(request)
+
+    def test_validate_whitespace_url(self):
+        """Test validation with whitespace-only URL."""
+        with pytest.raises(ValueError, match="URL cannot be empty"):
+            request = CrawlRequest(url="   ")
+            _validate_crawl_request(request)
+
+    def test_validate_valid_url(self):
+        """Test validation with valid URL."""
+        request = CrawlRequest(url="https://example.com")
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_invalid_limit(self):
+        """Test validation with invalid limit."""
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            request = CrawlRequest(
+                url="https://example.com",
+                limit=0
+            )
+            _validate_crawl_request(request)
+
+    def test_validate_negative_limit(self):
+        """Test validation with negative limit."""
+        with pytest.raises(ValueError, match="Limit must be positive"):
+            request = CrawlRequest(
+                url="https://example.com",
+                limit=-5
+            )
+            _validate_crawl_request(request)
+
+    def test_validate_valid_limit(self):
+        """Test validation with valid limit."""
+        request = CrawlRequest(
+            url="https://example.com",
+            limit=10
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_with_prompt(self):
+        """Test validation with prompt."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts"
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_with_prompt_and_options(self):
+        """Test validation with prompt and options."""
+        request = CrawlRequest(
+            url="https://example.com",
+            prompt="Extract all blog posts",
+            limit=10
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_none_options(self):
+        """Test validation with None options."""
+        request = CrawlRequest(url="https://example.com")
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_complex_options(self):
+        """Test validation with complex options."""
+        scrape_opts = ScrapeOptions(
+            formats=["markdown"],
+            only_main_content=False,
+            mobile=True
+        )
+        
+        request = CrawlRequest(
+            url="https://example.com",
+            limit=50,
+            max_discovery_depth=3,
+            scrape_options=scrape_opts
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+    def test_validate_scrape_options_integration(self):
+        """Test that scrape_options validation is integrated."""
+        # Test with valid scrape options
+        scrape_opts = ScrapeOptions(formats=["markdown"], timeout=30000)
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=scrape_opts
+        )
+        _validate_crawl_request(request)  # Should not raise
+
+        # Test with invalid scrape options (should raise error)
+        invalid_scrape_opts = ScrapeOptions(timeout=-1000)
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=invalid_scrape_opts
+        )
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            _validate_crawl_request(request) 

firecrawl/__tests__/unit/v2/methods/test_map_request_preparation.py

@@ -0,0 +1,54 @@
+import pytest
+from firecrawl.v2.types import MapOptions
+from firecrawl.v2.methods.map import _prepare_map_request
+
+
+class TestMapRequestPreparation:
+    """Unit tests for map request preparation."""
+
+    def test_basic_request_preparation(self):
+        data = _prepare_map_request("https://example.com")
+        assert data["url"] == "https://example.com"
+        # Default sitemap handling should be "include" when no flags provided
+        assert "sitemap" not in data  # we only send when options provided
+
+    def test_sitemap_transformations(self):
+        # sitemap -> "only"
+        opts = MapOptions(sitemap="only")
+        data = _prepare_map_request("https://example.com", opts)
+        assert data["sitemap"] == "only"
+
+        # sitemap -> "skip"
+        opts = MapOptions(sitemap="skip")
+        data = _prepare_map_request("https://example.com", opts)
+        assert data["sitemap"] == "skip"
+
+        # default when options present but sitemap left as default -> include
+        opts = MapOptions(search="docs")
+        data = _prepare_map_request("https://example.com", opts)
+        assert data["sitemap"] == "include"
+
+    def test_field_conversions(self):
+        opts = MapOptions(
+            search="docs",
+            include_subdomains=True,
+            limit=25,
+            sitemap="only",
+            timeout=15000,
+            integration="  _unit-test  ",
+        )
+        data = _prepare_map_request("https://example.com", opts)
+
+        assert data["url"] == "https://example.com"
+        assert data["search"] == "docs"
+        assert data["includeSubdomains"] is True
+        assert data["limit"] == 25
+        assert data["sitemap"] == "only"
+        assert data["timeout"] == 15000
+        assert data["integration"] == "_unit-test"
+
+    def test_invalid_url(self):
+        with pytest.raises(ValueError):
+            _prepare_map_request("")
+        with pytest.raises(ValueError):
+            _prepare_map_request("   ")