firecrawl 2.1.1__py3-none-any.whl → 2.1.2__py3-none-any.whl

build/lib/tests/test_change_tracking.py

@@ -0,0 +1,98 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import json
+import os
+from firecrawl import FirecrawlApp
+
+class TestChangeTracking(unittest.TestCase):
+    @patch('requests.post')
+    def test_change_tracking_format(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test markdown content',
+                'changeTracking': {
+                    'previousScrapeAt': '2023-01-01T00:00:00Z',
+                    'changeStatus': 'changed',
+                    'visibility': 'visible'
+                }
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        result = app.scrape_url('https://example.com', {
+            'formats': ['markdown', 'changeTracking']
+        })
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
+        
+        self.assertEqual(result['changeTracking']['previousScrapeAt'], '2023-01-01T00:00:00Z')
+        self.assertEqual(result['changeTracking']['changeStatus'], 'changed')
+        self.assertEqual(result['changeTracking']['visibility'], 'visible')
+
+    @patch('requests.post')
+    def test_change_tracking_options(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'success': True,
+            'data': {
+                'markdown': 'Test markdown content',
+                'changeTracking': {
+                    'previousScrapeAt': '2023-01-01T00:00:00Z',
+                    'changeStatus': 'changed',
+                    'visibility': 'visible',
+                    'diff': {
+                        'text': '@@ -1,1 +1,1 @@\n-old content\n+new content',
+                        'json': {
+                            'files': [{
+                                'from': None,
+                                'to': None,
+                                'chunks': [{
+                                    'content': '@@ -1,1 +1,1 @@',
+                                    'changes': [{
+                                        'type': 'del',
+                                        'content': '-old content',
+                                        'del': True,
+                                        'ln': 1
+                                    }, {
+                                        'type': 'add',
+                                        'content': '+new content',
+                                        'add': True,
+                                        'ln': 1
+                                    }]
+                                }]
+                            }]
+                        }
+                    },
+                    'json': {
+                        'title': {
+                            'previous': 'Old Title',
+                            'current': 'New Title'
+                        }
+                    }
+                }
+            }
+        }
+        mock_post.return_value = mock_response
+
+        app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
+        result = app.scrape_url('https://example.com', {
+            'formats': ['markdown', 'changeTracking'],
+            'changeTrackingOptions': {
+                'modes': ['git-diff', 'json'],
+                'schema': {'type': 'object', 'properties': {'title': {'type': 'string'}}}
+            }
+        })
+
+        args, kwargs = mock_post.call_args
+        self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
+        self.assertEqual(kwargs['json']['changeTrackingOptions']['modes'], ['git-diff', 'json'])
+        
+        self.assertEqual(result['changeTracking']['diff']['text'], '@@ -1,1 +1,1 @@\n-old content\n+new content')
+        self.assertEqual(result['changeTracking']['json']['title']['previous'], 'Old Title')
+        self.assertEqual(result['changeTracking']['json']['title']['current'], 'New Title')

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp, JsonConfig, ScrapeOptions # noqa
 
-__version__ = "2.1.1"
+__version__ = "2.1.2"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/firecrawl.py

@@ -1849,24 +1849,33 @@ class FirecrawlApp:
             show_full_text=show_full_text,
             experimental_stream=experimental_stream
         )
-        if not response.get('success') or 'id' not in response:
-            return response
+        
+        if not response.success or not response.id:
+            return GenerateLLMsTextStatusResponse(
+                success=False,
+                error='Failed to start LLMs.txt generation',
+                status='failed',
+                expiresAt=''
+            )
 
-        job_id = response['id']
+        job_id = response.id
         while True:
             status = self.check_generate_llms_text_status(job_id)
             
-            if status['status'] == 'completed':
+            if status.status == 'completed':
                 return status
-            elif status['status'] == 'failed':
-                raise Exception(f'LLMs.txt generation failed. Error: {status.get("error")}')
-            elif status['status'] != 'processing':
-                break
+            elif status.status == 'failed':
+                return status
+            elif status.status != 'processing':
+                return GenerateLLMsTextStatusResponse(
+                    success=False,
+                    error='LLMs.txt generation job terminated unexpectedly',
+                    status='failed',
+                    expiresAt=''
+                )
 
             time.sleep(2)  # Polling interval
 
-        return {'success': False, 'error': 'LLMs.txt generation job terminated unexpectedly'}
-
     def async_generate_llms_text(
             self,
             url: str,
@@ -1903,10 +1912,13 @@ class FirecrawlApp:
         json_data['origin'] = f"python-sdk@{version}"
 
         try:
-            response = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)
-            if response.status_code == 200:
+            req = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)
+            response = req.json()
+            print("json_data", json_data)
+            print("response", response)
+            if response.get('success'):
                 try:
-                    return response.json()
+                    return GenerateLLMsTextResponse(**response)
                 except:
                     raise Exception('Failed to parse Firecrawl response as JSON.')
             else:
@@ -1914,7 +1926,10 @@ class FirecrawlApp:
         except Exception as e:
             raise ValueError(str(e))
 
-        return {'success': False, 'error': 'Internal server error'}
+        return GenerateLLMsTextResponse(
+            success=False,
+            error='Internal server error'
+        )
 
     def check_generate_llms_text_status(self, id: str) -> GenerateLLMsTextStatusResponse:
         """
@@ -1941,9 +1956,10 @@ class FirecrawlApp:
             response = self._get_request(f'{self.api_url}/v1/llmstxt/{id}', headers)
             if response.status_code == 200:
                 try:
-                    return response.json()
-                except:
-                    raise Exception('Failed to parse Firecrawl response as JSON.')
+                    json_data = response.json()
+                    return GenerateLLMsTextStatusResponse(**json_data)
+                except Exception as e:
+                    raise Exception(f'Failed to parse Firecrawl response as GenerateLLMsTextStatusResponse: {str(e)}')
             elif response.status_code == 404:
                 raise Exception('LLMs.txt generation job not found')
             else:
@@ -1951,7 +1967,7 @@ class FirecrawlApp:
         except Exception as e:
             raise ValueError(str(e))
 
-        return {'success': False, 'error': 'Internal server error'}
+        return GenerateLLMsTextStatusResponse(success=False, error='Internal server error', status='failed', expiresAt='')
 
     def _prepare_headers(
             self,

{firecrawl-2.1.1.dist-info → firecrawl-2.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 2.1.1
+Version: 2.1.2
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai

firecrawl-2.1.2.dist-info/RECORD

@@ -0,0 +1,26 @@
+build/lib/build/lib/firecrawl/__init__.py,sha256=ULa8YPZxYyfqSWUKB3kuqDpCsteI9xpEEgoQhUDikvg,2570
+build/lib/build/lib/firecrawl/firecrawl.py,sha256=_TpfO8ltda563MIg1NbtPWY1DiWRRAqM2NBcW7DmHXM,178339
+build/lib/build/lib/firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+build/lib/build/lib/firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
+build/lib/build/lib/firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+build/lib/build/lib/firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=DcCw-cohtnL-t9XPekUtRoQrgg3UCWu8Ikqudf9ory8,19880
+build/lib/build/lib/tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+build/lib/firecrawl/__init__.py,sha256=ULa8YPZxYyfqSWUKB3kuqDpCsteI9xpEEgoQhUDikvg,2570
+build/lib/firecrawl/firecrawl.py,sha256=_TpfO8ltda563MIg1NbtPWY1DiWRRAqM2NBcW7DmHXM,178339
+build/lib/firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+build/lib/firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
+build/lib/firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+build/lib/firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=DcCw-cohtnL-t9XPekUtRoQrgg3UCWu8Ikqudf9ory8,19880
+build/lib/tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+firecrawl/__init__.py,sha256=ULa8YPZxYyfqSWUKB3kuqDpCsteI9xpEEgoQhUDikvg,2570
+firecrawl/firecrawl.py,sha256=_TpfO8ltda563MIg1NbtPWY1DiWRRAqM2NBcW7DmHXM,178339
+firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
+firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=DcCw-cohtnL-t9XPekUtRoQrgg3UCWu8Ikqudf9ory8,19880
+tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+firecrawl-2.1.2.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl-2.1.2.dist-info/METADATA,sha256=-RpAnHd-RpdV0syOIhYWzfprdpBZGibZQkdG-CHceDA,10583
+firecrawl-2.1.2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl-2.1.2.dist-info/top_level.txt,sha256=ytN_R30g2U2qZYFyIm710Z8QeK9FO1Uwa-WPGHXyqjE,27
+firecrawl-2.1.2.dist-info/RECORD,,

{firecrawl-2.1.1.dist-info → firecrawl-2.1.2.dist-info}/top_level.txt

@@ -1,2 +1,4 @@
+build
+dist
 firecrawl
 tests

firecrawl-2.1.1.dist-info/RECORD

@@ -1,12 +0,0 @@
-firecrawl/__init__.py,sha256=NU9Qcom12t48ym3ovFMpCYI4-uH-Ac1jnddqSUzxEIE,2570
-firecrawl/firecrawl.py,sha256=bXjJKt2UAdszpoCspBOPen_2lz5ysmVWP5vDMZUbyUo,177726
-firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-firecrawl/__tests__/e2e_withAuth/test.py,sha256=-Fq2vPcMo0iQi4dwsUkkCd931ybDaTxMBnZbRfGdDcA,7931
-firecrawl/__tests__/v1/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-firecrawl/__tests__/v1/e2e_withAuth/test.py,sha256=DcCw-cohtnL-t9XPekUtRoQrgg3UCWu8Ikqudf9ory8,19880
-tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
-firecrawl-2.1.1.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
-firecrawl-2.1.1.dist-info/METADATA,sha256=5_5qIPtR-xSv8jAkZLqBP1i-xefxucWl3rZo2OfPsLo,10583
-firecrawl-2.1.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-firecrawl-2.1.1.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
-firecrawl-2.1.1.dist-info/RECORD,,