firecrawl 0.0.16__py3-none-any.whl → 0.0.19__py3-none-any.whl

firecrawl/__init__.py

@@ -13,7 +13,7 @@ import os
 
 from .firecrawl import FirecrawlApp
 
-__version__ = "0.0.16"
+__version__ = "0.0.19"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

{firecrawl-0.0.16.dist-info → firecrawl-0.0.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl
-Version: 0.0.16
+Version: 0.0.19
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
@@ -30,7 +30,7 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Text Processing :: Indexing
-Requires-Python: >=3.8
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Requires-Dist: requests
 

firecrawl-0.0.19.dist-info/RECORD

@@ -0,0 +1,8 @@
+firecrawl/__init__.py,sha256=ixdimo55Kkf1ccxo31pemLUm16hBOG5GvgEen9S3qNc,1683
+firecrawl/firecrawl.py,sha256=wfy2KXSrLoGlj8HKEP3VKG_u38kRbUbcmRQsMgza5I0,12767
+firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+firecrawl/__tests__/e2e_withAuth/test.py,sha256=vZlhdURkMXY1-2_FBrKwT6D4WbH7HEf_xMEd0X63krQ,7665
+firecrawl-0.0.19.dist-info/METADATA,sha256=M92W_xzUtd6k5-789ti_yyfC88ZXZAdXq7Beqqx6iqc,6264
+firecrawl-0.0.19.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+firecrawl-0.0.19.dist-info/top_level.txt,sha256=jTvz79zWhiyAezfmmHe4FQ-hR60C59UU5FrjMjijLu8,10
+firecrawl-0.0.19.dist-info/RECORD,,

{firecrawl-0.0.16.dist-info → firecrawl-0.0.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

firecrawl-0.0.19.dist-info/top_level.txt

@@ -0,0 +1 @@
+firecrawl

build/lib/firecrawl/__init__.py

@@ -1,57 +0,0 @@
-"""
-This is the Firecrawl package.
-
-This package provides a Python SDK for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs.
-
-For more information visit https://github.com/firecrawl/
-"""
-
-import logging
-import os
-
-from .firecrawl import FirecrawlApp
-
-__version__ = "0.0.16"
-
-# Define the logger for the Firecrawl project
-logger: logging.Logger = logging.getLogger("firecrawl")
-
-
-def _basic_config() -> None:
-    """Set up basic configuration for logging with a specific format and date format."""
-    try:
-        logging.basicConfig(
-            format="[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-    except Exception as e:
-        logger.error("Failed to configure logging: %s", e)
-
-
-def setup_logging() -> None:
-    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    env = os.environ.get(
-        "FIRECRAWL_LOGGING_LEVEL", "INFO"
-    ).upper()  # Default to 'INFO' level
-    _basic_config()
-
-    if env == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif env == "INFO":
-        logger.setLevel(logging.INFO)
-    elif env == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif env == "ERROR":
-        logger.setLevel(logging.ERROR)
-    elif env == "CRITICAL":
-        logger.setLevel(logging.CRITICAL)
-    else:
-        logger.setLevel(logging.INFO)
-        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-
-# Initialize logging configuration when the module is imported
-setup_logging()
-logger.debug("Debugging logger setup")

build/lib/firecrawl/firecrawl.py

@@ -1,318 +0,0 @@
-"""
-FirecrawlApp Module
-
-This module provides a class `FirecrawlApp` for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs. The module uses requests for HTTP communication
-and handles retries for certain HTTP status codes.
-
-Classes:
-    - FirecrawlApp: Main class for interacting with the Firecrawl API.
-"""
-import logging
-import os
-import time
-from typing import Any, Dict, Optional
-
-import requests
-
-logger : logging.Logger = logging.getLogger("firecrawl")
-
-class FirecrawlApp:
-    """
-    Initialize the FirecrawlApp instance.
-
-    Args:
-        api_key (Optional[str]): API key for authenticating with the Firecrawl API.
-        api_url (Optional[str]): Base URL for the Firecrawl API.
-    """
-    def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
-        self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
-        if self.api_key is None:
-            logger.warning("No API key provided")
-            raise ValueError('No API key provided')
-        else:
-            logger.debug("Initialized FirecrawlApp with API key: %s", self.api_key)
-
-        self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')
-        if self.api_url != 'https://api.firecrawl.dev':
-            logger.debug("Initialized FirecrawlApp with API URL: %s", self.api_url)
-
-    def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any:
-        """
-        Scrape the specified URL using the Firecrawl API.
-
-        Args:
-            url (str): The URL to scrape.
-            params (Optional[Dict[str, Any]]): Additional parameters for the scrape request.
-
-        Returns:
-            Any: The scraped data if the request is successful.
-
-        Raises:
-            Exception: If the scrape request fails.
-        """
-
-        headers = self._prepare_headers()
-
-        # Prepare the base scrape parameters with the URL
-        scrape_params = {'url': url}
-
-        # If there are additional params, process them
-        if params:
-            # Initialize extractorOptions if present
-            extractor_options = params.get('extractorOptions', {})
-            # Check and convert the extractionSchema if it's a Pydantic model
-            if 'extractionSchema' in extractor_options:
-                if hasattr(extractor_options['extractionSchema'], 'schema'):
-                    extractor_options['extractionSchema'] = extractor_options['extractionSchema'].schema()
-                # Ensure 'mode' is set, defaulting to 'llm-extraction' if not explicitly provided
-                extractor_options['mode'] = extractor_options.get('mode', 'llm-extraction')
-                # Update the scrape_params with the processed extractorOptions
-                scrape_params['extractorOptions'] = extractor_options
-
-            # Include any other params directly at the top level of scrape_params
-            for key, value in params.items():
-                if key != 'extractorOptions':
-                    scrape_params[key] = value
-        # Make the POST request with the prepared headers and JSON data
-        response = requests.post(
-            f'{self.api_url}/v0/scrape',
-            headers=headers,
-            json=scrape_params,
-        )
-        if response.status_code == 200:
-            response = response.json()
-            if response['success'] and 'data' in response:
-                return response['data']
-            else:
-                raise Exception(f'Failed to scrape URL. Error: {response["error"]}')
-        else:
-            self._handle_error(response, 'scrape URL')
-
-    def search(self, query: str, params: Optional[Dict[str, Any]] = None) -> Any:
-        """
-        Perform a search using the Firecrawl API.
-
-        Args:
-            query (str): The search query.
-            params (Optional[Dict[str, Any]]): Additional parameters for the search request.
-
-        Returns:
-            Any: The search results if the request is successful.
-
-        Raises:
-            Exception: If the search request fails.
-        """
-        headers = self._prepare_headers()
-        json_data = {'query': query}
-        if params:
-            json_data.update(params)
-        response = requests.post(
-            f'{self.api_url}/v0/search',
-            headers=headers,
-            json=json_data
-        )
-        if response.status_code == 200:
-            response = response.json()
-
-            if response['success'] and 'data' in response:
-                return response['data']
-            else:
-                raise Exception(f'Failed to search. Error: {response["error"]}')
-
-        else:
-            self._handle_error(response, 'search')
-
-    def crawl_url(self, url: str,
-                  params: Optional[Dict[str, Any]] = None,
-                  wait_until_done: bool = True,
-                  poll_interval: int = 2,
-                  idempotency_key: Optional[str] = None) -> Any:
-        """
-        Initiate a crawl job for the specified URL using the Firecrawl API.
-
-        Args:
-            url (str): The URL to crawl.
-            params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
-            wait_until_done (bool): Whether to wait until the crawl job is completed.
-            poll_interval (int): Time in seconds between status checks when waiting for job completion.
-            idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
-
-        Returns:
-            Any: The crawl job ID or the crawl results if waiting until completion.
-
-        Raises:
-            Exception: If the crawl job initiation or monitoring fails.
-        """
-        headers = self._prepare_headers(idempotency_key)
-        json_data = {'url': url}
-        if params:
-            json_data.update(params)
-        response = self._post_request(f'{self.api_url}/v0/crawl', json_data, headers)
-        if response.status_code == 200:
-            job_id = response.json().get('jobId')
-            if wait_until_done:
-                return self._monitor_job_status(job_id, headers, poll_interval)
-            else:
-                return {'jobId': job_id}
-        else:
-            self._handle_error(response, 'start crawl job')
-
-    def check_crawl_status(self, job_id: str) -> Any:
-        """
-        Check the status of a crawl job using the Firecrawl API.
-
-        Args:
-            job_id (str): The ID of the crawl job.
-
-        Returns:
-            Any: The status of the crawl job.
-
-        Raises:
-            Exception: If the status check request fails.
-        """
-        headers = self._prepare_headers()
-        response = self._get_request(f'{self.api_url}/v0/crawl/status/{job_id}', headers)
-        if response.status_code == 200:
-            return response.json()
-        else:
-            self._handle_error(response, 'check crawl status')
-
-    def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
-        """
-        Prepare the headers for API requests.
-
-        Args:
-            idempotency_key (Optional[str]): A unique key to ensure idempotency of requests.
-
-        Returns:
-            Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.
-        """
-        if idempotency_key:
-            return {
-                'Content-Type': 'application/json',
-                'Authorization': f'Bearer {self.api_key}',
-                'x-idempotency-key': idempotency_key
-            }
-
-        return {
-            'Content-Type': 'application/json',
-            'Authorization': f'Bearer {self.api_key}',
-        }
-
-    def _post_request(self, url: str,
-                      data: Dict[str, Any],
-                      headers: Dict[str, str],
-                      retries: int = 3,
-                      backoff_factor: float = 0.5) -> requests.Response:
-        """
-        Make a POST request with retries.
-
-        Args:
-            url (str): The URL to send the POST request to.
-            data (Dict[str, Any]): The JSON data to include in the POST request.
-            headers (Dict[str, str]): The headers to include in the POST request.
-            retries (int): Number of retries for the request.
-            backoff_factor (float): Backoff factor for retries.
-
-        Returns:
-            requests.Response: The response from the POST request.
-
-        Raises:
-            requests.RequestException: If the request fails after the specified retries.
-        """
-        for attempt in range(retries):
-            response = requests.post(url, headers=headers, json=data)
-            if response.status_code == 502:
-                time.sleep(backoff_factor * (2 ** attempt))
-            else:
-                return response
-        return response
-
-    def _get_request(self, url: str,
-                     headers: Dict[str, str],
-                     retries: int = 3,
-                     backoff_factor: float = 0.5) -> requests.Response:
-        """
-        Make a GET request with retries.
-
-        Args:
-            url (str): The URL to send the GET request to.
-            headers (Dict[str, str]): The headers to include in the GET request.
-            retries (int): Number of retries for the request.
-            backoff_factor (float): Backoff factor for retries.
-
-        Returns:
-            requests.Response: The response from the GET request.
-
-        Raises:
-            requests.RequestException: If the request fails after the specified retries.
-        """
-        for attempt in range(retries):
-            response = requests.get(url, headers=headers)
-            if response.status_code == 502:
-                time.sleep(backoff_factor * (2 ** attempt))
-            else:
-                return response
-        return response
-
-    def _monitor_job_status(self, job_id: str, headers: Dict[str, str], poll_interval: int) -> Any:
-        """
-        Monitor the status of a crawl job until completion.
-
-        Args:
-            job_id (str): The ID of the crawl job.
-            headers (Dict[str, str]): The headers to include in the status check requests.
-            poll_interval (int): Secounds between status checks.
-
-        Returns:
-            Any: The crawl results if the job is completed successfully.
-
-        Raises:
-            Exception: If the job fails or an error occurs during status checks.
-        """
-        while True:
-            status_response = self._get_request(f'{self.api_url}/v0/crawl/status/{job_id}', headers)
-            if status_response.status_code == 200:
-                status_data = status_response.json()
-                if status_data['status'] == 'completed':
-                    if 'data' in status_data:
-                        return status_data['data']
-                    else:
-                        raise Exception('Crawl job completed but no data was returned')
-                elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting']:
-                    poll_interval=max(poll_interval,2)
-                    time.sleep(poll_interval)  # Wait for the specified interval before checking again
-                else:
-                    raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}')
-            else:
-                self._handle_error(status_response, 'check crawl status')
-
-    def _handle_error(self, response: requests.Response, action: str) -> None:
-        """
-        Handle errors from API responses.
-
-        Args:
-            response (requests.Response): The response object from the API request.
-            action (str): Description of the action that was being performed.
-
-        Raises:
-            Exception: An exception with a message containing the status code and error details from the response.
-        """
-        error_message = response.json().get('error', 'No additional error details provided.')
-
-        if response.status_code == 402:
-            message = f"Payment Required: Failed to {action}. {error_message}"
-        elif response.status_code == 408:
-            message = f"Request Timeout: Failed to {action} as the request timed out. {error_message}"
-        elif response.status_code == 409:
-            message = f"Conflict: Failed to {action} due to a conflict. {error_message}"
-        elif response.status_code == 500:
-            message = f"Internal Server Error: Failed to {action}. {error_message}"
-        else:
-            message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message}"
-
-        # Raise an HTTPError with the custom message and attach the response
-        raise requests.exceptions.HTTPError(message, response=response)
-    

firecrawl-0.0.16.dist-info/RECORD

@@ -1,10 +0,0 @@
-build/lib/firecrawl/__init__.py,sha256=QRMTTelCkQdvP5R8Z1k71Xo9FFORks1WqT0jW4HxAbg,1683
-build/lib/firecrawl/firecrawl.py,sha256=wfy2KXSrLoGlj8HKEP3VKG_u38kRbUbcmRQsMgza5I0,12767
-firecrawl/__init__.py,sha256=QRMTTelCkQdvP5R8Z1k71Xo9FFORks1WqT0jW4HxAbg,1683
-firecrawl/firecrawl.py,sha256=wfy2KXSrLoGlj8HKEP3VKG_u38kRbUbcmRQsMgza5I0,12767
-firecrawl/__tests__/e2e_withAuth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-firecrawl/__tests__/e2e_withAuth/test.py,sha256=vZlhdURkMXY1-2_FBrKwT6D4WbH7HEf_xMEd0X63krQ,7665
-firecrawl-0.0.16.dist-info/METADATA,sha256=xecqvrVvCCDsy5e53Uik80_tamWXrPoJlGHtZbDcVkU,6264
-firecrawl-0.0.16.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-firecrawl-0.0.16.dist-info/top_level.txt,sha256=GglqAyOiUmWSqRivafO_tbV_rz2OdmRzMG9BRS3A6Zo,21
-firecrawl-0.0.16.dist-info/RECORD,,

firecrawl-0.0.16.dist-info/top_level.txt

@@ -1,3 +0,0 @@
-build
-dist
-firecrawl