firecrawl 4.12.0__py3-none-any.whl

firecrawl/__tests__/unit/v2/utils/test_validation.py

@@ -0,0 +1,311 @@
+import pytest
+from firecrawl.v2.types import JsonFormat, ScrapeOptions, PDFParser
+from firecrawl.v2.utils.validation import validate_scrape_options, prepare_scrape_options
+
+
+class TestValidateScrapeOptions:
+    """Unit tests for validate_scrape_options function."""
+
+    def test_validate_none_options(self):
+        """Test validation with None options."""
+        result = validate_scrape_options(None)
+        assert result is None
+
+    def test_validate_valid_options(self):
+        """Test validation with valid options."""
+        options = ScrapeOptions(
+            formats=["markdown"],
+            timeout=30000,
+            wait_for=2000
+        )
+        result = validate_scrape_options(options)
+        assert result == options
+
+    def test_validate_invalid_timeout(self):
+        """Test validation with invalid timeout."""
+        options = ScrapeOptions(timeout=0)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            validate_scrape_options(options)
+
+    def test_validate_negative_timeout(self):
+        """Test validation with negative timeout."""
+        options = ScrapeOptions(timeout=-1000)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            validate_scrape_options(options)
+
+    def test_validate_invalid_wait_for(self):
+        """Test validation with invalid wait_for."""
+        options = ScrapeOptions(wait_for=-500)
+        with pytest.raises(ValueError, match="wait_for must be non-negative"):
+            validate_scrape_options(options)
+
+    def test_validate_zero_wait_for(self):
+        """Test validation with zero wait_for (should be valid)."""
+        options = ScrapeOptions(wait_for=0)
+        result = validate_scrape_options(options)
+        assert result == options
+
+    def test_validate_complex_options(self):
+        """Test validation with complex options."""
+        options = ScrapeOptions(
+            formats=["markdown", "html"],
+            headers={"User-Agent": "Test"},
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False,
+            raw_html=True,
+            screenshot_full_page=True
+        )
+        result = validate_scrape_options(options)
+        assert result == options
+
+    def test_validate_multiple_invalid_fields(self):
+        """Test validation with multiple invalid fields."""
+        options = ScrapeOptions(timeout=-1000, wait_for=-500)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            validate_scrape_options(options)
+        # Should fail on first invalid field (timeout)
+
+    def test_validate_edge_cases(self):
+        """Test validation with edge case values."""
+        # Test with very large timeout
+        options = ScrapeOptions(timeout=999999)
+        result = validate_scrape_options(options)
+        assert result == options
+
+        # Test with very large wait_for
+        options = ScrapeOptions(wait_for=999999)
+        result = validate_scrape_options(options)
+        assert result == options
+
+
+class TestPrepareScrapeOptions:
+    """Unit tests for prepare_scrape_options function."""
+
+    def test_prepare_none_options(self):
+        """Test preparation with None options."""
+        result = prepare_scrape_options(None)
+        assert result is None
+
+    def test_prepare_basic_options(self):
+        """Test preparation with basic options."""
+        options = ScrapeOptions(
+            formats=["markdown"],
+            timeout=30000,
+            wait_for=2000
+        )
+        result = prepare_scrape_options(options)
+        
+        assert isinstance(result, dict)
+        assert "formats" in result
+        assert "timeout" in result
+        assert "waitFor" in result
+        assert result["timeout"] == 30000
+        assert result["waitFor"] == 2000
+
+    def test_prepare_snake_case_conversion(self):
+        """Test snake_case to camelCase conversion."""
+        options = ScrapeOptions(
+            include_tags=["h1", "h2"],
+            exclude_tags=["nav"],
+            only_main_content=False,
+            wait_for=2000,
+            skip_tls_verification=True,
+            remove_base64_images=False
+            # Note: raw_html should be in formats array, not as a separate field
+        )
+        result = prepare_scrape_options(options)
+    
+        # Check conversions
+        assert "includeTags" in result
+        assert result["includeTags"] == ["h1", "h2"]
+        assert "excludeTags" in result
+        assert result["excludeTags"] == ["nav"]
+        assert "onlyMainContent" in result
+        assert result["onlyMainContent"] is False
+        assert "waitFor" in result
+        assert result["waitFor"] == 2000
+        assert "skipTlsVerification" in result
+        assert result["skipTlsVerification"] is True
+        assert "removeBase64Images" in result
+        assert result["removeBase64Images"] is False
+        
+        # Check that snake_case fields are not present
+        assert "include_tags" not in result
+        assert "exclude_tags" not in result
+        assert "only_main_content" not in result
+        assert "wait_for" not in result
+        assert "skip_tls_verification" not in result
+        assert "remove_base64_images" not in result
+
+    def test_prepare_complex_options(self):
+        """Test preparation with complex options."""
+        options = ScrapeOptions(
+            formats=["markdown", "html", "rawHtml"],
+            headers={"User-Agent": "Test Bot"},
+            include_tags=["h1", "h2", "h3"],
+            exclude_tags=["nav", "footer"],
+            only_main_content=False,
+            timeout=15000,
+            wait_for=2000,
+            mobile=True,
+            skip_tls_verification=True,
+            remove_base64_images=False
+        )
+        result = prepare_scrape_options(options)
+        
+        # Check all fields are present and converted
+        assert "formats" in result
+        assert "headers" in result
+        assert "includeTags" in result
+        assert "excludeTags" in result
+        assert "onlyMainContent" in result
+        assert "timeout" in result
+        assert "waitFor" in result
+        assert "mobile" in result
+        assert "skipTlsVerification" in result
+        assert "removeBase64Images" in result
+        
+        # Check values
+        assert result["formats"] == ["markdown", "html", "rawHtml"]
+        assert result["headers"] == {"User-Agent": "Test Bot"}
+        assert result["includeTags"] == ["h1", "h2", "h3"]
+        assert result["excludeTags"] == ["nav", "footer"]
+        assert result["onlyMainContent"] is False
+        assert result["timeout"] == 15000
+        assert result["waitFor"] == 2000
+        assert result["mobile"] is True
+        assert result["skipTlsVerification"] is True
+        assert result["removeBase64Images"] is False
+
+    def test_prepare_invalid_options(self):
+        """Test preparation with invalid options (should raise error)."""
+        options = ScrapeOptions(timeout=-1000)
+        with pytest.raises(ValueError, match="Timeout must be positive"):
+            prepare_scrape_options(options)
+
+    def test_prepare_empty_options(self):
+        """Test preparation with empty options."""
+        options = ScrapeOptions()  # All defaults
+        result = prepare_scrape_options(options)
+        
+        # Should return dict with default values
+        assert isinstance(result, dict)
+        assert "onlyMainContent" in result
+        assert result["onlyMainContent"] is True
+        assert "mobile" in result
+        assert result["mobile"] is False
+
+    def test_prepare_none_values(self):
+        """Test preparation with None values in options."""
+        options = ScrapeOptions(
+            formats=None,
+            timeout=None,
+            wait_for=None,
+            include_tags=None,
+            exclude_tags=None
+        )
+        result = prepare_scrape_options(options)
+        
+        # Should only include non-None values
+        assert isinstance(result, dict)
+        # Should have default values for required fields
+        assert "onlyMainContent" in result
+        assert "mobile" in result 
+
+    def test_format_schema_conversion(self):
+        """Test that Format schema is properly handled."""
+        # Create a JsonFormat object with schema
+        format_obj = JsonFormat(
+            type="json",
+            prompt="Extract product info",
+            schema={"type": "object", "properties": {"name": {"type": "string"}}}
+        )
+        
+        dumped = format_obj.model_dump()
+        assert "schema" in dumped
+        assert dumped["schema"] == {"type": "object", "properties": {"name": {"type": "string"}}} 
+
+    def test_prepare_new_v2_fields(self):
+        """Test preparation with new v2 fields."""
+        from firecrawl.v2.types import Viewport, ScreenshotAction
+        
+        viewport = Viewport(width=1920, height=1080)
+        screenshot_action = ScreenshotAction(
+            type="screenshot",
+            full_page=True,
+            quality=90,
+            viewport=viewport
+        )
+        
+        options = ScrapeOptions(
+            fast_mode=True,
+            use_mock="test-mock",
+            block_ads=False,
+            store_in_cache=False,
+            max_age=7200000,  # 2 hours
+            actions=[screenshot_action],
+            parsers=["pdf"]
+        )
+        
+        result = prepare_scrape_options(options)
+        
+        # Check new field conversions
+        assert "fastMode" in result
+        assert result["fastMode"] is True
+        assert "useMock" in result
+        assert result["useMock"] == "test-mock"
+        assert "blockAds" in result
+        assert result["blockAds"] is False
+        assert "storeInCache" in result
+        assert result["storeInCache"] is False
+        assert "maxAge" in result
+        assert result["maxAge"] == 7200000
+        
+        # Check actions conversion
+        assert "actions" in result
+        assert len(result["actions"]) == 1
+        action = result["actions"][0]
+        assert action["type"] == "screenshot"
+        assert action["fullPage"] is True
+        assert action["quality"] == 90
+        assert "viewport" in action
+        assert action["viewport"]["width"] == 1920
+        assert action["viewport"]["height"] == 1080
+        
+        # Check parsers
+        assert "parsers" in result
+        assert result["parsers"] == ["pdf"]
+        
+        # Check that snake_case fields are not present
+        assert "fast_mode" not in result
+        assert "use_mock" not in result
+        assert "block_ads" not in result
+        assert "store_in_cache" not in result
+        assert "max_age" not in result 
+
+    def test_prepare_parsers_max_pages_dict(self):
+        """Ensure parser dicts convert max_pages to maxPages."""
+        options = ScrapeOptions(
+            parsers=[{"type": "pdf", "max_pages": 3}]
+        )
+
+        result = prepare_scrape_options(options)
+
+        assert "parsers" in result
+        assert result["parsers"][0]["maxPages"] == 3
+        assert "max_pages" not in result["parsers"][0]
+
+    def test_prepare_parsers_max_pages_model(self):
+        """Ensure parser models convert max_pages to maxPages."""
+        parser = PDFParser(max_pages=5)
+        options = ScrapeOptions(parsers=[parser])
+
+        result = prepare_scrape_options(options)
+
+        assert result["parsers"][0]["maxPages"] == 5

firecrawl/__tests__/unit/v2/watcher/test_ws_watcher.py

@@ -0,0 +1,332 @@
+import asyncio
+import json
+import time
+import pytest
+
+from firecrawl.v2.watcher import Watcher
+
+
+class DummyHttpClient:
+    def __init__(self, api_url: str = "http://localhost", api_key: str = "TEST"):
+        self.api_url = api_url
+        self.api_key = api_key
+
+
+class DummyClient:
+    def __init__(self):
+        self.http_client = DummyHttpClient()
+
+
+class FakeWebSocket:
+    def __init__(self, messages):
+        # messages: list of dicts to be json-dumped
+        self._messages = list(messages)
+
+    async def recv(self):
+        if not self._messages:
+            # No more messages; block a bit to allow loop to end
+            await asyncio.sleep(0.01)
+            # Simulate disconnect
+            raise asyncio.CancelledError()
+        msg = self._messages.pop(0)
+        return json.dumps(msg)
+
+
+class FakeConnect:
+    def __init__(self, ws: FakeWebSocket):
+        self._ws = ws
+
+    async def __aenter__(self):
+        return self._ws
+
+    async def __aexit__(self, exc_type, exc, tb):
+        return False
+
+
+@pytest.mark.parametrize("kind", ["crawl", "batch"]) 
+def test_ws_watcher_document_and_done(monkeypatch, kind):
+    # Prepare messages: one document then done
+    messages = [
+        {"type": "document", "data": {"url": "https://example.com", "rawHtml": "<html>"}},
+        {"type": "done", "data": {"status": "completed", "data": []}},
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    def fake_connect(uri, *args, **kwargs):
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind=kind)
+
+    events = {"document": 0, "done": 0}
+    statuses = []
+
+    watcher.add_event_listener("document", lambda d: events.__setitem__("document", events["document"] + 1))
+    watcher.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
+    watcher.add_listener(lambda s: statuses.append(s.status))
+
+    watcher.start()
+
+    # Wait for thread to finish
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert events["document"] >= 1
+    assert events["done"] == 1
+    assert statuses[-1] in ("completed", "failed", "cancelled")
+
+
+def test_ws_watcher_error_event(monkeypatch):
+    messages = [
+        {"type": "error", "error": "boom", "data": {"status": "failed"}},
+    ]
+    ws = FakeWebSocket(messages)
+
+    def fake_connect(uri, *args, **kwargs):
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind="crawl")
+
+    seen_error = {"count": 0}
+    watcher.add_event_listener("error", lambda d: seen_error.__setitem__("count", seen_error["count"] + 1))
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert seen_error["count"] == 1
+
+
+@pytest.mark.parametrize("kind", ["crawl", "batch"]) 
+def test_ws_watcher_catchup_dispatches_documents_and_updates_status(monkeypatch, kind):
+    messages = [
+        {
+            "type": "catchup",
+            "data": {
+                "status": "scraping",
+                "data": [
+                    {"url": "https://example.com/1", "rawHtml": "<html>1</html>"},
+                    {"url": "https://example.com/2", "rawHtml": "<html>2</html>"},
+                ],
+            },
+        }
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    def fake_connect(uri, *args, **kwargs):
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind=kind)
+
+    events = {"document": 0}
+    statuses = []
+
+    watcher.add_event_listener("document", lambda d: events.__setitem__("document", events["document"] + 1))
+    watcher.add_listener(lambda s: statuses.append(s.status))
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert events["document"] == 2
+    assert statuses[-1] == "scraping"
+
+
+@pytest.mark.parametrize("kind", ["crawl", "batch"]) 
+def test_ws_watcher_status_only_terminal_snapshot_triggers_done(monkeypatch, kind):
+    # No explicit type, only a terminal status snapshot
+    messages = [
+        {"data": {"status": "completed", "data": []}},
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    def fake_connect(uri, *args, **kwargs):
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind=kind)
+
+    events = {"done": 0}
+    statuses = []
+
+    watcher.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
+    watcher.add_listener(lambda s: statuses.append(s.status))
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert events["done"] == 1
+    assert statuses[-1] == "completed"
+
+
+def test_ws_watcher_batch_cancelled_snapshot_no_done_event(monkeypatch):
+    # Batch-only: cancelled snapshot should end without a 'done' event
+    messages = [
+        {"data": {"status": "cancelled", "data": []}},
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    def fake_connect(uri, *args, **kwargs):
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind="batch")
+
+    events = {"done": 0}
+    statuses = []
+
+    watcher.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
+    watcher.add_listener(lambda s: statuses.append(s.status))
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert events["done"] == 0
+    assert statuses[-1] == "cancelled"
+
+
+def test_ws_watcher_propagates_authorization_header(monkeypatch):
+    # Ensure Authorization header is forwarded to websockets.connect
+    messages = [
+        {"type": "done", "data": {"status": "completed", "data": []}},
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    captured_headers = {"headers": None}
+
+    def fake_connect(uri, *args, **kwargs):
+        captured_headers["headers"] = kwargs.get("additional_headers")
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind="crawl")
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert captured_headers["headers"] is not None
+    # Expect an Authorization header with Bearer token
+    assert any(h[0] == "Authorization" and "Bearer" in h[1] for h in captured_headers["headers"]) 
+
+
+@pytest.mark.parametrize("kind", ["crawl", "batch"]) 
+def test_ws_watcher_normalizes_document_fields_in_snapshot(monkeypatch, kind):
+    # Status-only snapshot with camelCase fields should be normalized in emitted job
+    messages = [
+        {"data": {"status": "completed", "data": [
+            {"url": "https://example.com/x", "rawHtml": "<x>", "changeTracking": {"modes": ["git-diff"]}}
+        ]}},
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    def fake_connect(uri, *args, **kwargs):
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind=kind)
+
+    jobs = []
+    watcher.add_listener(lambda j: jobs.append(j))
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert jobs, "No job snapshots emitted"
+    last_job = jobs[-1]
+    assert last_job.status == "completed"
+    assert last_job.data and last_job.data[0].raw_html == "<x>"
+    assert last_job.data[0].change_tracking is not None
+
+
+@pytest.mark.parametrize("kind", ["crawl", "batch"]) 
+def test_ws_watcher_uses_correct_ws_uri(monkeypatch, kind):
+    # Verify WS URI uses the correct path per kind and http->ws scheme
+    messages = [
+        {"type": "done", "data": {"status": "completed", "data": []}},
+    ]
+
+    ws = FakeWebSocket(messages)
+
+    captured_uri = {"uri": None}
+
+    def fake_connect(uri, *args, **kwargs):
+        captured_uri["uri"] = uri
+        return FakeConnect(ws)
+
+    import websockets
+    monkeypatch.setattr(websockets, "connect", fake_connect)
+
+    client = DummyClient()
+    watcher = Watcher(client, job_id="jid", kind=kind)
+
+    watcher.start()
+
+    deadline = time.time() + 2
+    while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
+        time.sleep(0.01)
+
+    watcher.stop()
+
+    assert captured_uri["uri"] is not None
+    expected = "ws://localhost/v2/crawl/jid" if kind == "crawl" else "ws://localhost/v2/batch/scrape/jid"
+    assert captured_uri["uri"] == expected