firecrawl-py 3.2.1__py3-none-any.whl → 3.3.1__py3-none-any.whl

build/lib/firecrawl/v2/watcher_async.py

@@ -0,0 +1,242 @@
+"""
+Async WebSocket watcher with async iterator interface for v2 jobs.
+
+Usage:
+    async for snapshot in AsyncWatcher(client, job_id, kind="crawl"):
+        print(snapshot.status)
+"""
+
+import asyncio
+import inspect
+import json
+import time
+from typing import AsyncIterator, Dict, List, Literal, Optional
+
+import websockets
+from websockets.exceptions import ConnectionClosed, ConnectionClosedOK, ConnectionClosedError
+
+from .types import BatchScrapeJob, CrawlJob, Document
+from .utils.normalize import normalize_document_input
+
+JobKind = Literal["crawl", "batch"]
+
+
+class AsyncWatcher:
+    def __init__(
+        self,
+        client: object,
+        job_id: str,
+        *,
+        kind: JobKind = "crawl",
+        timeout: Optional[int] = None,
+    ) -> None:
+        self._client = client
+        self._job_id = job_id
+        self._kind = kind
+        self._timeout = timeout
+        self._poll_interval: float = 2.0
+
+        http_client = getattr(client, "http_client", None)
+        if http_client is not None:
+            self._api_url = getattr(http_client, "api_url", None)
+            self._api_key = getattr(http_client, "api_key", None)
+        else:
+            # Allow passing the top-level Firecrawl client directly
+            self._api_url = getattr(client, "api_url", None)
+            self._api_key = getattr(client, "api_key", None)
+
+        self._status: str = "scraping"
+        self._data: List[Dict] = []
+
+    def __aiter__(self) -> AsyncIterator[object]:
+        return self._iterate()
+
+    def _build_ws_url(self) -> str:
+        if not self._api_url:
+            raise ValueError("API URL is required for WebSocket watcher")
+        ws_base = self._api_url.replace("https://", "wss://").replace("http://", "ws://", 1)
+        if self._kind == "crawl":
+            return f"{ws_base}/v2/crawl/{self._job_id}"
+        return f"{ws_base}/v2/batch/scrape/{self._job_id}"
+
+    async def _iterate(self) -> AsyncIterator[object]:
+        uri = self._build_ws_url()
+        headers_list = []
+        if self._api_key:
+            headers_list.append(("Authorization", f"Bearer {self._api_key}"))
+
+        # Attempt to establish WS; on failure, fall back to HTTP polling immediately
+        try:
+            async with websockets.connect(uri, max_size=None, additional_headers=headers_list) as websocket:
+                deadline = asyncio.get_event_loop().time() + self._timeout if self._timeout else None
+                # Pre-yield a snapshot if available to ensure progress is visible
+                try:
+                    pre = await self._fetch_job_status()
+                    yield pre
+                    if pre.status in ("completed", "failed", "cancelled"):
+                        return
+                except Exception:
+                    pass
+
+                while True:
+                    try:
+                        if deadline is not None:
+                            remaining = max(0.0, deadline - asyncio.get_event_loop().time())
+                            timeout = min(self._poll_interval, remaining) if remaining > 0 else 0.0
+                        else:
+                            timeout = self._poll_interval
+                        msg = await asyncio.wait_for(websocket.recv(), timeout=timeout)
+                    except asyncio.TimeoutError:
+                        # Quiet period: poll HTTP once
+                        job = await self._safe_fetch()
+                        if job is not None:
+                            yield job
+                            if job.status in ("completed", "failed", "cancelled"):
+                                return
+                        if deadline is not None and asyncio.get_event_loop().time() >= deadline:
+                            return
+                        continue
+                    except (ConnectionClosedOK, ConnectionClosed, ConnectionClosedError):
+                        # Graceful/abrupt close: poll HTTP until terminal (bounded by timeout)
+                        deadline = time.time() + (self._timeout or 30)
+                        while True:
+                            try:
+                                job = await self._fetch_job_status()
+                                yield job
+                                if job.status in ("completed", "failed", "cancelled"):
+                                    return
+                            except Exception:
+                                return
+                            if time.time() >= deadline:
+                                return
+                            await asyncio.sleep(1)
+                    try:
+                        body = json.loads(msg)
+                    except Exception:
+                        continue
+
+                    msg_type = body.get("type")
+                    if msg_type == "error":
+                        self._status = "failed"
+                        # Yield a terminal snapshot
+                        if self._kind == "crawl":
+                            yield CrawlJob(status="failed", completed=0, total=0, credits_used=0, expires_at=None, next=None, data=[])
+                        else:
+                            yield BatchScrapeJob(status="failed", completed=0, total=0, credits_used=0, expires_at=None, next=None, data=[])
+                        return
+                    elif msg_type == "catchup":
+                        d = body.get("data", {})
+                        self._status = d.get("status", self._status)
+                        docs_in = d.get("data", []) or []
+                        self._data.extend(docs_in)
+                        # Fall through to emit a snapshot below
+                    elif msg_type == "document":
+                        doc = body.get("data")
+                        if isinstance(doc, dict):
+                            self._data.append(doc)
+                        # Fall through to emit a snapshot below
+                    elif msg_type == "done":
+                        self._status = "completed"
+                        raw_payload = body.get("data", {}) or {}
+                        docs_in = raw_payload.get("data", []) or []
+                        if isinstance(docs_in, list) and docs_in:
+                            for doc in docs_in:
+                                if isinstance(doc, dict):
+                                    self._data.append(doc)
+                        # Emit final snapshot then end
+                        yield self._make_snapshot(status="completed", payload=raw_payload, docs_override=self._data)
+                        return
+
+                    # Generic snapshot emit for status messages and periodic progress
+                    payload = body.get("data", body)
+                    status_str = payload.get("status", body.get("status", self._status))
+                    snapshot = self._make_snapshot(status=status_str, payload=payload)
+                    yield snapshot
+                    if status_str in ("completed", "failed", "cancelled"):
+                        return
+        except Exception:
+            # WS connect failure: fallback to HTTP polling loop until terminal/timeout
+            deadline = time.time() + (self._timeout or 30)
+            while True:
+                try:
+                    job = await self._fetch_job_status()
+                    yield job
+                    if job.status in ("completed", "failed", "cancelled"):
+                        return
+                except Exception:
+                    return
+                if time.time() >= deadline:
+                    return
+                await asyncio.sleep(1)
+
+    async def _fetch_job_status(self):
+        if self._kind == "crawl":
+            return await self._call_status_method("get_crawl_status")
+        return await self._call_status_method("get_batch_scrape_status")
+
+    async def _call_status_method(self, method_name: str):
+        # Try on client directly
+        meth = getattr(self._client, method_name, None)
+        if meth is not None:
+            try:
+                result = meth(self._job_id)
+            except TypeError:
+                result = None
+            if result is not None:
+                if inspect.isawaitable(result):
+                    return await result
+                return result
+            # Fallback: if we couldn't call directly, try to_thread
+            return await asyncio.to_thread(meth, self._job_id)
+
+        # Try on client.v2
+        v2 = getattr(self._client, "v2", None)
+        if v2 is not None:
+            meth = getattr(v2, method_name, None)
+            if meth is not None:
+                try:
+                    result = meth(self._job_id)
+                except TypeError:
+                    result = None
+                if result is not None:
+                    if inspect.isawaitable(result):
+                        return await result
+                    return result
+                return await asyncio.to_thread(meth, self._job_id)
+
+        raise RuntimeError(f"Client does not expose {method_name}")
+
+    async def _safe_fetch(self):
+        try:
+            return await self._fetch_job_status()
+        except Exception:
+            return None
+
+    def _make_snapshot(self, *, status: str, payload: Dict, docs_override: Optional[List[Dict]] = None):
+        docs = []
+        source_docs = docs_override if docs_override is not None else payload.get("data", []) or []
+        for doc in source_docs:
+            if isinstance(doc, dict):
+                d = normalize_document_input(doc)
+                docs.append(Document(**d))
+
+        if self._kind == "crawl":
+            return CrawlJob(
+                status=status,
+                completed=payload.get("completed", 0),
+                total=payload.get("total", 0),
+                credits_used=payload.get("creditsUsed", 0),
+                expires_at=payload.get("expiresAt"),
+                next=payload.get("next"),
+                data=docs,
+            )
+        return BatchScrapeJob(
+            status=status,
+            completed=payload.get("completed", 0),
+            total=payload.get("total", 0),
+            credits_used=payload.get("creditsUsed"),
+            expires_at=payload.get("expiresAt"),
+            next=payload.get("next"),
+            data=docs,
+        )
+

build/lib/tests/test_change_tracking.py

@@ -0,0 +1,98 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import json
+import os
+from firecrawl import FirecrawlApp
+
+class TestChangeTracking(unittest.TestCase):
+    @patch('requests.post')
+    def test_change_tracking_format(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test markdown content',
+                'changeTracking': {
+                    'previousScrapeAt': '2023-01-01T00:00:00Z',
+                    'changeStatus': 'changed',
+                    'visibility': 'visible'
+                }
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        result = app.scrape_url('https://example.com', {
+            'formats': ['markdown', 'changeTracking']
+        })
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
+        
+        self.assertEqual(result['changeTracking']['previousScrapeAt'], '2023-01-01T00:00:00Z')
+        self.assertEqual(result['changeTracking']['changeStatus'], 'changed')
+        self.assertEqual(result['changeTracking']['visibility'], 'visible')
+
+    @patch('requests.post')
+    def test_change_tracking_options(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test markdown content',
+                'changeTracking': {
+                    'previousScrapeAt': '2023-01-01T00:00:00Z',
+                    'changeStatus': 'changed',
+                    'visibility': 'visible',
+                    'diff': {
+                        'text': '@@ -1,1 +1,1 @@\n-old content\n+new content',
+                        'json': {
+                            'files': [{
+                                'from': None,
+                                'to': None,
+                                'chunks': [{
+                                    'content': '@@ -1,1 +1,1 @@',
+                                    'changes': [{
+                                        'type': 'del',
+                                        'content': '-old content',
+                                        'del': True,
+                                        'ln': 1
+                                    }, {
+                                        'type': 'add',
+                                        'content': '+new content',
+                                        'add': True,
+                                        'ln': 1
+                                    }]
+                                }]
+                            }]
+                        }
+                    },
+                    'json': {
+                        'title': {
+                            'previous': 'Old Title',
+                            'current': 'New Title'
+                        }
+                    }
+                }
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        result = app.scrape_url('https://example.com', {
+            'formats': ['markdown', 'changeTracking'],
+            'changeTrackingOptions': {
+                'modes': ['git-diff', 'json'],
+                'schema': {'type': 'object', 'properties': {'title': {'type': 'string'}}}
+            }
+        })
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
+        self.assertEqual(kwargs['json']['changeTrackingOptions']['modes'], ['git-diff', 'json'])
+        
+        self.assertEqual(result['changeTracking']['diff']['text'], '@@ -1,1 +1,1 @@\n-old content\n+new content')
+        self.assertEqual(result['changeTracking']['json']['title']['previous'], 'Old Title')
+        self.assertEqual(result['changeTracking']['json']['title']['current'], 'New Title')

build/lib/tests/test_timeout_conversion.py

@@ -0,0 +1,117 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import os
+from firecrawl import FirecrawlApp
+
+
+class TestTimeoutConversion(unittest.TestCase):
+    
+    @patch('requests.post')
+    def test_scrape_url_timeout_conversion(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test content'
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        app.scrape_url('https://example.com', timeout=60000)
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['timeout'], 65.0)
+
+    @patch('requests.post')
+    def test_scrape_url_default_timeout(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test content'
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        app.scrape_url('https://example.com')
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['timeout'], 35.0)
+
+    @patch('requests.post')
+    def test_post_request_timeout_conversion(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        
+        data = {'timeout': 30000}
+        headers = {'Content-Type': 'application/json'}
+        
+        app._post_request('https://example.com/api', data, headers)
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['timeout'], 35.0)
+
+    @patch('requests.post')
+    def test_post_request_default_timeout(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        
+        data = {'timeout': 30000, 'url': 'https://example.com'}
+        headers = {'Content-Type': 'application/json'}
+        
+        app._post_request('https://example.com/api', data, headers)
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['timeout'], 35.0)
+
+    @patch('requests.post')
+    def test_timeout_edge_cases(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test content'
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        
+        app.scrape_url('https://example.com', timeout=1000)
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['timeout'], 6.0)
+        
+        app.scrape_url('https://example.com', timeout=0)
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['timeout'], 5.0)
+
+    @patch('requests.post')
+    def test_post_request_no_timeout_key(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        
+        data = {'url': 'https://example.com'}
+        headers = {'Content-Type': 'application/json'}
+        
+        app._post_request('https://example.com/api', data, headers)
+
+        args, kwargs = mock_post.call_args
+        self.assertIsNone(kwargs['timeout'])
+
+
+if __name__ == '__main__':
+    unittest.main()

firecrawl/__init__.py

@@ -17,7 +17,7 @@ from .v1 import (
     V1ChangeTrackingOptions,
 )
 
-__version__ = "3.2.1"
+__version__ = "3.3.1"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_request_preparation.py

@@ -14,7 +14,7 @@ class TestAsyncCrawlRequestPreparation:
             include_paths=["/docs/*"],
             exclude_paths=["/admin/*"],
             max_discovery_depth=2,
-            ignore_sitemap=True,
+            sitemap="skip",
             ignore_query_parameters=True,
             crawl_entire_domain=True,
             allow_external_links=False,
@@ -26,7 +26,7 @@ class TestAsyncCrawlRequestPreparation:
         assert payload["includePaths"] == ["/docs/*"]
         assert payload["excludePaths"] == ["/admin/*"]
         assert payload["maxDiscoveryDepth"] == 2
-        assert payload["ignoreSitemap"] is True
+        assert payload["sitemap"] == "skip"
         assert payload["ignoreQueryParameters"] is True
         assert payload["crawlEntireDomain"] is True
         assert payload["allowExternalLinks"] is False

firecrawl/__tests__/unit/v2/methods/test_crawl_request_preparation.py

@@ -24,7 +24,7 @@ class TestCrawlRequestPreparation:
             url="https://example.com",
             limit=10,
             max_discovery_depth=3,
-            ignore_sitemap=True,
+            sitemap="skip",
             crawl_entire_domain=False,
             allow_external_links=True
         )
@@ -39,8 +39,8 @@ class TestCrawlRequestPreparation:
         assert data["limit"] == 10
         assert "maxDiscoveryDepth" in data
         assert data["maxDiscoveryDepth"] == 3
-        assert "ignoreSitemap" in data
-        assert data["ignoreSitemap"] is True
+        assert "sitemap" in data
+        assert data["sitemap"] == "skip"
         assert "crawlEntireDomain" in data
         assert data["crawlEntireDomain"] is False
         assert "allowExternalLinks" in data
@@ -106,7 +106,7 @@ class TestCrawlRequestPreparation:
             include_paths=["/blog/*", "/docs/*"],
             exclude_paths=["/admin/*"],
             max_discovery_depth=3,
-            ignore_sitemap=False,
+            sitemap="include",
             limit=100,
             crawl_entire_domain=True,
             allow_external_links=False,
@@ -126,8 +126,8 @@ class TestCrawlRequestPreparation:
         assert data["excludePaths"] == ["/admin/*"]
         assert "maxDiscoveryDepth" in data
         assert data["maxDiscoveryDepth"] == 3
-        assert "ignoreSitemap" in data
-        assert data["ignoreSitemap"] is False
+        assert "sitemap" in data
+        assert data["sitemap"] == "include"
         assert "limit" in data
         assert data["limit"] == 100
         assert "crawlEntireDomain" in data

firecrawl/v2/client.py

@@ -13,6 +13,7 @@ from .types import (
     SearchRequest,
     SearchData,
     SourceOption,
+    CategoryOption,
     CrawlRequest,
     CrawlResponse,
     CrawlJob,
@@ -171,6 +172,7 @@ class FirecrawlClient:
         query: str,
         *,
         sources: Optional[List[SourceOption]] = None,
+        categories: Optional[List[CategoryOption]] = None,
         limit: Optional[int] = None,
         tbs: Optional[str] = None,
         location: Optional[str] = None,
@@ -195,6 +197,7 @@ class FirecrawlClient:
         request = SearchRequest(
             query=query,
             sources=sources,
+            categories=categories,
             limit=limit,
             tbs=tbs,
             location=location,

firecrawl/v2/methods/search.py

@@ -121,6 +121,17 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:
                 if source.type not in valid_sources:
                     raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
     
+    # Validate categories (if provided)
+    if request.categories is not None:
+        valid_categories = {"github", "research"}
+        for category in request.categories:
+            if isinstance(category, str):
+                if category not in valid_categories:
+                    raise ValueError(f"Invalid category type: {category}. Valid types: {valid_categories}")
+            elif hasattr(category, 'type'):
+                if category.type not in valid_categories:
+                    raise ValueError(f"Invalid category type: {category.type}. Valid types: {valid_categories}")
+    
     # Validate location (if provided)
     if request.location is not None:
         if not isinstance(request.location, str) or len(request.location.strip()) == 0:

firecrawl/v2/types.py

@@ -174,6 +174,12 @@ class Source(BaseModel):
 
 SourceOption = Union[str, Source]
 
+class Category(BaseModel):
+    """Configuration for a search category."""
+    type: str
+
+CategoryOption = Union[str, Category]
+
 FormatString = Literal[
     # camelCase versions (API format)
     "markdown", "html", "rawHtml", "links", "screenshot", "summary", "changeTracking", "json",
@@ -331,7 +337,8 @@ class SearchResultWeb(BaseModel):
     """A web search result with URL, title, and description."""
     url: str
     title: Optional[str] = None
-    description: Optional[str] = None 
+    description: Optional[str] = None
+    category: Optional[str] = None
 
 class SearchResultNews(BaseModel):
   """A news search result with URL, title, snippet, date, image URL, and position."""
@@ -341,6 +348,7 @@ class SearchResultNews(BaseModel):
   date: Optional[str] = None
   image_url: Optional[str] = None
   position: Optional[int] = None
+  category: Optional[str] = None
 
 class SearchResultImages(BaseModel):
   """An image search result with URL, title, image URL, image width, image height, and position."""
@@ -521,6 +529,7 @@ class SearchRequest(BaseModel):
     """Request for search operations."""
     query: str
     sources: Optional[List[SourceOption]] = None
+    categories: Optional[List[CategoryOption]] = None
     limit: Optional[int] = 5
     tbs: Optional[str] = None
     location: Optional[str] = None
@@ -547,6 +556,26 @@ class SearchRequest(BaseModel):
                 raise ValueError(f"Invalid source format: {source}")
         
         return normalized_sources
+    
+    @field_validator('categories')
+    @classmethod
+    def validate_categories(cls, v):
+        """Validate and normalize categories input."""
+        if v is None:
+            return v
+        
+        normalized_categories = []
+        for category in v:
+            if isinstance(category, str):
+                normalized_categories.append(Category(type=category))
+            elif isinstance(category, dict):
+                normalized_categories.append(Category(**category))
+            elif isinstance(category, Category):
+                normalized_categories.append(category)
+            else:
+                raise ValueError(f"Invalid category format: {category}")
+        
+        return normalized_categories
 
 class LinkResult(BaseModel):
     """A generic link result with optional metadata (used by search and map)."""

{firecrawl_py-3.2.1.dist-info → firecrawl_py-3.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: firecrawl-py
-Version: 3.2.1
+Version: 3.3.1
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/firecrawl/firecrawl
 Author: Mendable.ai
@@ -38,12 +38,8 @@ Requires-Dist: httpx
 Requires-Dist: python-dotenv
 Requires-Dist: websockets
 Requires-Dist: nest-asyncio
-Requires-Dist: pydantic>=2.0
+Requires-Dist: pydantic (>=2.0)
 Requires-Dist: aiohttp
-Dynamic: author
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-python
 
 # Firecrawl Python SDK