firecrawl 1.14.1__tar.gz → 1.16.0__tar.gz

{firecrawl-1.14.1 → firecrawl-1.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 1.14.1
+Version: 1.16.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

{firecrawl-1.14.1 → firecrawl-1.16.0}/firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp # noqa
 
-__version__ = "1.14.1"
+__version__ = "1.16.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

{firecrawl-1.14.1 → firecrawl-1.16.0}/firecrawl/firecrawl.py

@@ -48,6 +48,8 @@ class DeepResearchParams(pydantic.BaseModel):
     maxDepth: Optional[int] = 7
     timeLimit: Optional[int] = 270
     maxUrls: Optional[int] = 20
+    analysisPrompt: Optional[str] = None
+    systemPrompt: Optional[str] = None
     __experimental_streamSteps: Optional[bool] = None
 
 class DeepResearchResponse(pydantic.BaseModel):
@@ -73,6 +75,16 @@ class DeepResearchStatusResponse(pydantic.BaseModel):
     sources: List[Dict[str, Any]]
     summaries: List[str]
 
+class ChangeTrackingData(pydantic.BaseModel):
+    """
+    Data for the change tracking format.
+    """
+    previousScrapeAt: Optional[str] = None
+    changeStatus: str  # "new" | "same" | "changed" | "removed"
+    visibility: str  # "visible" | "hidden"
+    diff: Optional[Dict[str, Any]] = None
+    json: Optional[Any] = None
+
 class FirecrawlApp:
     class SearchResponse(pydantic.BaseModel):
         """
@@ -165,9 +177,13 @@ class FirecrawlApp:
                     json['schema'] = json['schema'].schema()
                 scrape_params['jsonOptions'] = json
 
+            change_tracking = params.get("changeTrackingOptions", {})
+            if change_tracking:
+                scrape_params['changeTrackingOptions'] = change_tracking
+
             # Include any other params directly at the top level of scrape_params
             for key, value in params.items():
-                if key not in ['jsonOptions']:
+                if key not in ['jsonOptions', 'changeTrackingOptions']:
                     scrape_params[key] = value
 
 
@@ -1098,6 +1114,8 @@ class FirecrawlApp:
 
         if response.status_code == 402:
             message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
+        elif response.status_code == 403:
+            message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
         elif response.status_code == 408:
             message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
         elif response.status_code == 409:
@@ -1168,7 +1186,6 @@ class FirecrawlApp:
             time.sleep(2)  # Polling interval
 
         return {'success': False, 'error': 'Deep research job terminated unexpectedly'}
-
     def async_deep_research(self, query: str, params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None) -> Dict[str, Any]:
         """
         Initiates an asynchronous deep research operation.
@@ -1192,8 +1209,15 @@ class FirecrawlApp:
             research_params = params
 
         headers = self._prepare_headers()
+        
         json_data = {'query': query, **research_params.dict(exclude_none=True)}
 
+        # Handle json options schema if present
+        if 'jsonOptions' in json_data:
+            json_opts = json_data['jsonOptions']
+            if json_opts and 'schema' in json_opts and hasattr(json_opts['schema'], 'schema'):
+                json_data['jsonOptions']['schema'] = json_opts['schema'].schema()
+
         try:
             response = self._post_request(f'{self.api_url}/v1/deep-research', json_data, headers)
             if response.status_code == 200:

{firecrawl-1.14.1 → firecrawl-1.16.0}/firecrawl.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 1.14.1
+Version: 1.16.0
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

{firecrawl-1.14.1 → firecrawl-1.16.0}/firecrawl.egg-info/SOURCES.txt

@@ -12,4 +12,5 @@ firecrawl.egg-info/top_level.txt
 firecrawl/__tests__/e2e_withAuth/__init__.py
 firecrawl/__tests__/e2e_withAuth/test.py
 firecrawl/__tests__/v1/e2e_withAuth/__init__.py
-firecrawl/__tests__/v1/e2e_withAuth/test.py
+firecrawl/__tests__/v1/e2e_withAuth/test.py
+tests/test_change_tracking.py

{firecrawl-1.14.1 → firecrawl-1.16.0}/firecrawl.egg-info/top_level.txt

@@ -1 +1,2 @@
 firecrawl
+tests

firecrawl-1.16.0/tests/test_change_tracking.py

@@ -0,0 +1,98 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import json
+import os
+from firecrawl import FirecrawlApp
+
+class TestChangeTracking(unittest.TestCase):
+    @patch('requests.post')
+    def test_change_tracking_format(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test markdown content',
+                'changeTracking': {
+                    'previousScrapeAt': '2023-01-01T00:00:00Z',
+                    'changeStatus': 'changed',
+                    'visibility': 'visible'
+                }
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        result = app.scrape_url('https://example.com', {
+            'formats': ['markdown', 'changeTracking']
+        })
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
+        
+        self.assertEqual(result['changeTracking']['previousScrapeAt'], '2023-01-01T00:00:00Z')
+        self.assertEqual(result['changeTracking']['changeStatus'], 'changed')
+        self.assertEqual(result['changeTracking']['visibility'], 'visible')
+
+    @patch('requests.post')
+    def test_change_tracking_options(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test markdown content',
+                'changeTracking': {
+                    'previousScrapeAt': '2023-01-01T00:00:00Z',
+                    'changeStatus': 'changed',
+                    'visibility': 'visible',
+                    'diff': {
+                        'text': '@@ -1,1 +1,1 @@\n-old content\n+new content',
+                        'json': {
+                            'files': [{
+                                'from': None,
+                                'to': None,
+                                'chunks': [{
+                                    'content': '@@ -1,1 +1,1 @@',
+                                    'changes': [{
+                                        'type': 'del',
+                                        'content': '-old content',
+                                        'del': True,
+                                        'ln': 1
+                                    }, {
+                                        'type': 'add',
+                                        'content': '+new content',
+                                        'add': True,
+                                        'ln': 1
+                                    }]
+                                }]
+                            }]
+                        }
+                    },
+                    'json': {
+                        'title': {
+                            'previous': 'Old Title',
+                            'current': 'New Title'
+                        }
+                    }
+                }
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        result = app.scrape_url('https://example.com', {
+            'formats': ['markdown', 'changeTracking'],
+            'changeTrackingOptions': {
+                'modes': ['git-diff', 'json'],
+                'schema': {'type': 'object', 'properties': {'title': {'type': 'string'}}}
+            }
+        })
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
+        self.assertEqual(kwargs['json']['changeTrackingOptions']['modes'], ['git-diff', 'json'])
+        
+        self.assertEqual(result['changeTracking']['diff']['text'], '@@ -1,1 +1,1 @@\n-old content\n+new content')
+        self.assertEqual(result['changeTracking']['json']['title']['previous'], 'Old Title')
+        self.assertEqual(result['changeTracking']['json']['title']['current'], 'New Title')