firecrawl 4.12.0__py3-none-any.whl

firecrawl/__tests__/unit/v2/methods/test_pagination.py

@@ -0,0 +1,671 @@
+"""
+Unit tests for Firecrawl v2 pagination functionality.
+"""
+
+import pytest
+import time
+from unittest.mock import Mock, patch, AsyncMock
+from typing import Dict, Any, List
+
+from firecrawl.v2.types import (
+    PaginationConfig, 
+    CrawlJob, 
+    BatchScrapeJob, 
+    Document, 
+    DocumentMetadata
+)
+from firecrawl.v2.methods.crawl import get_crawl_status, _fetch_all_pages
+from firecrawl.v2.methods.batch import get_batch_scrape_status, _fetch_all_batch_pages
+from firecrawl.v2.methods.aio.crawl import get_crawl_status as get_crawl_status_async, _fetch_all_pages_async
+from firecrawl.v2.methods.aio.batch import get_batch_scrape_status as get_batch_scrape_status_async, _fetch_all_batch_pages_async
+
+
+class TestPaginationConfig:
+    """Test PaginationConfig model."""
+    
+    def test_default_values(self):
+        """Test default values for PaginationConfig."""
+        config = PaginationConfig()
+        assert config.auto_paginate is True
+        assert config.max_pages is None
+        assert config.max_results is None
+        assert config.max_wait_time is None
+    
+    def test_custom_values(self):
+        """Test custom values for PaginationConfig."""
+        config = PaginationConfig(
+            auto_paginate=False,
+            max_pages=5,
+            max_results=100,
+            max_wait_time=30
+        )
+        assert config.auto_paginate is False
+        assert config.max_pages == 5
+        assert config.max_results == 100
+        assert config.max_wait_time == 30
+
+
+class TestCrawlPagination:
+    """Test crawl pagination functionality."""
+    
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.mock_client = Mock()
+        self.job_id = "test-crawl-123"
+        
+        # Sample document data
+        self.sample_doc = {
+            "url": "https://example.com",
+            "markdown": "# Test Content",
+            "metadata": {
+                "title": "Test Page",
+                "statusCode": 200
+            }
+        }
+    
+    def test_get_crawl_status_no_pagination(self):
+        """Test get_crawl_status with auto_paginate=False."""
+        # Mock response with next URL
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 10,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.return_value = mock_response
+        
+        # Test with auto_paginate=False
+        pagination_config = PaginationConfig(auto_paginate=False)
+        result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert result.status == "completed"
+        assert result.next == "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
+        assert len(result.data) == 1
+        assert isinstance(result.data[0], Document)
+
+    def test_get_crawl_status_propagates_request_timeout(self):
+        """Ensure request_timeout is forwarded to the HTTP client."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 1,
+            "total": 1,
+            "creditsUsed": 1,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": [self.sample_doc],
+        }
+
+        self.mock_client.get.return_value = mock_response
+
+        timeout_seconds = 5.5
+        import firecrawl.v2.methods.crawl as crawl_module
+
+        assert crawl_module.__file__.endswith("firecrawl/v2/methods/crawl.py")
+        assert crawl_module.get_crawl_status.__kwdefaults__ is not None
+        assert "request_timeout" in crawl_module.get_crawl_status.__kwdefaults__
+        result = get_crawl_status(
+            self.mock_client,
+            self.job_id,
+            request_timeout=timeout_seconds,
+        )
+
+        assert result.status == "completed"
+        self.mock_client.get.assert_called_with(
+            f"/v2/crawl/{self.job_id}", timeout=timeout_seconds
+        )
+    
+    def test_get_crawl_status_with_pagination(self):
+        """Test get_crawl_status with auto_paginate=True."""
+        # Mock first page response
+        mock_response1 = Mock()
+        mock_response1.ok = True
+        mock_response1.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 20,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        # Mock second page response
+        mock_response2 = Mock()
+        mock_response2.ok = True
+        mock_response2.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 20,
+            "total": 20,
+            "creditsUsed": 10,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Test with auto_paginate=True
+        pagination_config = PaginationConfig(auto_paginate=True)
+        result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert result.status == "completed"
+        assert result.next is None  # Should be None when auto_paginate=True
+        assert len(result.data) == 2
+        assert self.mock_client.get.call_count == 2
+    
+    def test_get_crawl_status_max_pages_limit(self):
+        """Test get_crawl_status with max_pages limit."""
+        # Mock responses for multiple pages
+        mock_responses = []
+        for i in range(5):  # 5 pages available
+            mock_response = Mock()
+            mock_response.ok = True
+            mock_response.json.return_value = {
+                "success": True,
+                "status": "completed",
+                "completed": (i + 1) * 10,
+                "total": 50,
+                "creditsUsed": (i + 1) * 5,
+                "expiresAt": "2024-01-01T00:00:00Z",
+                "next": f"https://api.firecrawl.dev/v2/crawl/test-crawl-123?page={i+2}" if i < 4 else None,
+                "data": [self.sample_doc]
+            }
+            mock_responses.append(mock_response)
+        
+        self.mock_client.get.side_effect = mock_responses
+        
+        # Test with max_pages=3
+        pagination_config = PaginationConfig(auto_paginate=True, max_pages=3)
+        result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert len(result.data) == 4  # 1 initial + 3 from pages
+        assert self.mock_client.get.call_count == 4  # 1 initial + 3 pagination calls
+    
+    def test_get_crawl_status_max_results_limit(self):
+        """Test get_crawl_status with max_results limit."""
+        # Mock responses with multiple documents per page
+        mock_response1 = Mock()
+        mock_response1.ok = True
+        mock_response1.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 20,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
+            "data": [self.sample_doc, self.sample_doc, self.sample_doc]  # 3 docs
+        }
+        
+        mock_response2 = Mock()
+        mock_response2.ok = True
+        mock_response2.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 20,
+            "total": 20,
+            "creditsUsed": 10,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3",
+            "data": [self.sample_doc, self.sample_doc]  # 2 more docs
+        }
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Test with max_results=4
+        pagination_config = PaginationConfig(auto_paginate=True, max_results=4)
+        result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert len(result.data) == 4  # Should stop at 4 results
+        assert self.mock_client.get.call_count == 2  # Should fetch 2 pages
+    
+    def test_get_crawl_status_max_wait_time_limit(self):
+        """Test get_crawl_status with max_wait_time limit."""
+        # Mock slow response
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 20,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.return_value = mock_response
+        
+        # Test with max_wait_time=1 second
+        pagination_config = PaginationConfig(auto_paginate=True, max_wait_time=1)
+        
+        with patch('firecrawl.v2.methods.crawl.time.monotonic', side_effect=[0, 2]):  # Simulate 2 seconds elapsed
+            result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert len(result.data) == 1  # Should stop due to timeout
+        assert self.mock_client.get.call_count == 1
+    
+    def test_fetch_all_pages_error_handling(self):
+        """Test _fetch_all_pages with API errors."""
+        # Mock first page success, second page error
+        mock_response1 = Mock()
+        mock_response1.ok = True
+        mock_response1.json.return_value = {
+            "success": True,
+            "data": [self.sample_doc],
+            "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
+        }
+        
+        mock_response2 = Mock()
+        mock_response2.ok = False
+        mock_response2.status_code = 500
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Should continue with what we have
+        result = _fetch_all_pages(self.mock_client, "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2", [], None)
+        
+        assert len(result) == 1  # Should have the first page data
+        assert self.mock_client.get.call_count == 2
+
+
+class TestBatchScrapePagination:
+    """Test batch scrape pagination functionality."""
+    
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.mock_client = Mock()
+        self.job_id = "test-batch-123"
+        
+        # Sample document data
+        self.sample_doc = {
+            "url": "https://example.com",
+            "markdown": "# Test Content",
+            "metadata": {
+                "title": "Test Page",
+                "statusCode": 200
+            }
+        }
+    
+    def test_get_batch_scrape_status_no_pagination(self):
+        """Test get_batch_scrape_status with auto_paginate=False."""
+        # Mock response with next URL
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 10,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.return_value = mock_response
+        
+        # Test with auto_paginate=False
+        pagination_config = PaginationConfig(auto_paginate=False)
+        result = get_batch_scrape_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert result.status == "completed"
+        assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
+        assert len(result.data) == 1
+        assert isinstance(result.data[0], Document)
+    
+    def test_get_batch_scrape_status_with_pagination(self):
+        """Test get_batch_scrape_status with auto_paginate=True."""
+        # Mock first page response
+        mock_response1 = Mock()
+        mock_response1.ok = True
+        mock_response1.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 20,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        # Mock second page response
+        mock_response2 = Mock()
+        mock_response2.ok = True
+        mock_response2.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 20,
+            "total": 20,
+            "creditsUsed": 10,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Test with auto_paginate=True
+        pagination_config = PaginationConfig(auto_paginate=True)
+        result = get_batch_scrape_status(self.mock_client, self.job_id, pagination_config)
+        
+        assert result.status == "completed"
+        assert result.next is None  # Should be None when auto_paginate=True
+        assert len(result.data) == 2
+        assert self.mock_client.get.call_count == 2
+    
+    def test_fetch_all_batch_pages_limits(self):
+        """Test _fetch_all_batch_pages with various limits."""
+        # Mock responses for multiple pages
+        mock_responses = []
+        for i in range(5):  # 5 pages available
+            mock_response = Mock()
+            mock_response.ok = True
+            mock_response.json.return_value = {
+                "success": True,
+                "data": [self.sample_doc, self.sample_doc],  # 2 docs per page
+                "next": f"https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page={i+2}" if i < 4 else None
+            }
+            mock_responses.append(mock_response)
+        
+        self.mock_client.get.side_effect = mock_responses
+        
+        # Test with max_pages=2, max_results=4 (total docs we want)
+        pagination_config = PaginationConfig(max_pages=2, max_results=4)
+        result = _fetch_all_batch_pages(
+            self.mock_client, 
+            "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2", 
+            [Document(**self.sample_doc)],  # 1 initial doc
+            pagination_config
+        )
+        
+        # Should have 1 initial + 3 from pages (limited by max_results=4)
+        assert len(result) == 4
+        assert self.mock_client.get.call_count == 2  # Should fetch 2 pages
+
+
+class TestAsyncPagination:
+    """Test async pagination functionality."""
+    
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.mock_client = AsyncMock()
+        self.job_id = "test-async-123"
+        
+        # Sample document data
+        self.sample_doc = {
+            "url": "https://example.com",
+            "markdown": "# Test Content",
+            "metadata": {
+                "title": "Test Page",
+                "statusCode": 200
+            }
+        }
+    
+    @pytest.mark.asyncio
+    async def test_get_crawl_status_async_with_pagination(self):
+        """Test async get_crawl_status with pagination."""
+        # Mock first page response
+        mock_response1 = Mock()
+        mock_response1.status_code = 200
+        mock_response1.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 20,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        # Mock second page response
+        mock_response2 = Mock()
+        mock_response2.status_code = 200
+        mock_response2.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 20,
+            "total": 20,
+            "creditsUsed": 10,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Test with auto_paginate=True
+        pagination_config = PaginationConfig(auto_paginate=True)
+        result = await get_crawl_status_async(self.mock_client, self.job_id, pagination_config)
+        
+        assert result.status == "completed"
+        assert result.next is None
+        assert len(result.data) == 2
+        assert self.mock_client.get.call_count == 2
+
+    @pytest.mark.asyncio
+    async def test_get_crawl_status_async_propagates_request_timeout(self):
+        """Ensure async request_timeout is forwarded to the HTTP client."""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 1,
+            "total": 1,
+            "creditsUsed": 1,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": [self.sample_doc],
+        }
+
+        self.mock_client.get.return_value = mock_response
+
+        timeout_seconds = 3.3
+        import firecrawl.v2.methods.aio.crawl as crawl_module_async
+
+        assert crawl_module_async.__file__.endswith("firecrawl/v2/methods/aio/crawl.py")
+        assert crawl_module_async.get_crawl_status.__kwdefaults__ is not None
+        assert "request_timeout" in crawl_module_async.get_crawl_status.__kwdefaults__
+        result = await get_crawl_status_async(
+            self.mock_client,
+            self.job_id,
+            request_timeout=timeout_seconds,
+        )
+
+        assert result.status == "completed"
+        self.mock_client.get.assert_awaited_with(
+            f"/v2/crawl/{self.job_id}", timeout=timeout_seconds
+        )
+
+    @pytest.mark.asyncio
+    async def test_get_batch_scrape_status_async_with_pagination(self):
+        """Test async get_batch_scrape_status with pagination."""
+        # Mock first page response
+        mock_response1 = Mock()
+        mock_response1.status_code = 200
+        mock_response1.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 10,
+            "total": 20,
+            "creditsUsed": 5,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2",
+            "data": [self.sample_doc]
+        }
+        
+        # Mock second page response
+        mock_response2 = Mock()
+        mock_response2.status_code = 200
+        mock_response2.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 20,
+            "total": 20,
+            "creditsUsed": 10,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": [self.sample_doc]
+        }
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Test with auto_paginate=True
+        pagination_config = PaginationConfig(auto_paginate=True)
+        result = await get_batch_scrape_status_async(self.mock_client, self.job_id, pagination_config)
+        
+        assert result.status == "completed"
+        assert result.next is None
+        assert len(result.data) == 2
+        assert self.mock_client.get.call_count == 2
+    
+    @pytest.mark.asyncio
+    async def test_fetch_all_pages_async_limits(self):
+        """Test async _fetch_all_pages_async with limits."""
+        # Mock responses for multiple pages
+        mock_responses = []
+        for i in range(3):  # 3 pages available
+            mock_response = Mock()
+            mock_response.status_code = 200
+            mock_response.json.return_value = {
+                "success": True,
+                "data": [self.sample_doc],
+                "next": f"https://api.firecrawl.dev/v2/crawl/test-async-123?page={i+2}" if i < 2 else None
+            }
+            mock_responses.append(mock_response)
+        
+        self.mock_client.get.side_effect = mock_responses
+        
+        # Test with max_pages=2
+        pagination_config = PaginationConfig(max_pages=2)
+        result = await _fetch_all_pages_async(
+            self.mock_client,
+            "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2",
+            [Document(**self.sample_doc)],  # 1 initial doc
+            pagination_config
+        )
+        
+        assert len(result) == 3  # 1 initial + 2 from pages
+        assert self.mock_client.get.call_count == 2
+
+
+class TestPaginationEdgeCases:
+    """Test pagination edge cases and error conditions."""
+    
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.mock_client = Mock()
+        self.sample_doc = {
+            "url": "https://example.com",
+            "markdown": "# Test Content",
+            "metadata": {"title": "Test Page"}
+        }
+    
+    def test_pagination_with_empty_data(self):
+        """Test pagination when API returns empty data."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 0,
+            "total": 0,
+            "creditsUsed": 0,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": []
+        }
+        
+        self.mock_client.get.return_value = mock_response
+        
+        pagination_config = PaginationConfig(auto_paginate=True)
+        result = get_crawl_status(self.mock_client, "test-123", pagination_config)
+        
+        assert len(result.data) == 0
+        assert result.next is None
+    
+    def test_pagination_with_string_data(self):
+        """Test pagination when API returns string data (should be skipped)."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": True,
+            "status": "completed",
+            "completed": 2,
+            "total": 2,
+            "creditsUsed": 1,
+            "expiresAt": "2024-01-01T00:00:00Z",
+            "next": None,
+            "data": ["https://example.com", self.sample_doc]  # String + dict
+        }
+        
+        self.mock_client.get.return_value = mock_response
+        
+        pagination_config = PaginationConfig(auto_paginate=True)
+        result = get_crawl_status(self.mock_client, "test-123", pagination_config)
+        
+        assert len(result.data) == 1  # Only the dict should be processed
+        assert isinstance(result.data[0], Document)
+    
+    def test_pagination_with_failed_response(self):
+        """Test pagination when API response indicates failure."""
+        mock_response = Mock()
+        mock_response.ok = True
+        mock_response.json.return_value = {
+            "success": False,
+            "error": "Job not found"
+        }
+        
+        self.mock_client.get.return_value = mock_response
+        
+        pagination_config = PaginationConfig(auto_paginate=True)
+        
+        with pytest.raises(Exception, match="Job not found"):
+            get_crawl_status(self.mock_client, "test-123", pagination_config)
+    
+    def test_pagination_with_unsuccessful_page(self):
+        """Test pagination when a subsequent page is unsuccessful."""
+        # Mock first page success
+        mock_response1 = Mock()
+        mock_response1.ok = True
+        mock_response1.json.return_value = {
+            "success": True,
+            "data": [self.sample_doc],
+            "next": "https://api.firecrawl.dev/v2/crawl/test-123?page=2"
+        }
+        
+        # Mock second page failure
+        mock_response2 = Mock()
+        mock_response2.ok = True
+        mock_response2.json.return_value = {
+            "success": False,
+            "error": "Page not found"
+        }
+        
+        self.mock_client.get.side_effect = [mock_response1, mock_response2]
+        
+        # Should continue with what we have
+        result = _fetch_all_pages(self.mock_client, "https://api.firecrawl.dev/v2/crawl/test-123?page=2", [], None)
+        
+        assert len(result) == 1  # Should have the first page data
+        assert self.mock_client.get.call_count == 2
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])