firecrawl-py 0.0.14__tar.gz → 4.10.5__tar.gz

firecrawl_py-4.10.5/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Sideguide Technologies Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

firecrawl_py-4.10.5/PKG-INFO

@@ -0,0 +1,234 @@
+Metadata-Version: 2.4
+Name: firecrawl-py
+Version: 4.10.5
+Summary: Python SDK for Firecrawl API
+Home-page: https://github.com/firecrawl/firecrawl
+Author: Mendable.ai
+Author-email: "Mendable.ai" <nick@mendable.ai>
+Maintainer-email: "Mendable.ai" <nick@mendable.ai>
+License: MIT License
+Project-URL: Documentation, https://docs.firecrawl.dev
+Project-URL: Source, https://github.com/firecrawl/firecrawl
+Project-URL: Tracker, https://github.com/firecrawl/firecrawl/issues
+Keywords: SDK,API,firecrawl
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Internet
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: httpx
+Requires-Dist: python-dotenv
+Requires-Dist: websockets
+Requires-Dist: nest-asyncio
+Requires-Dist: pydantic>=2.0
+Requires-Dist: aiohttp
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# Firecrawl Python SDK
+
+The Firecrawl Python SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
+
+## Installation
+
+To install the Firecrawl Python SDK, you can use pip:
+
+```bash 
+pip install firecrawl-py
+```
+
+## Usage
+
+1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
+2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `Firecrawl` class.
+
+Here's an example of how to use the SDK:
+
+```python 
+from firecrawl import Firecrawl
+from firecrawl.types import ScrapeOptions
+
+firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY")
+
+# Scrape a website (v2):
+data = firecrawl.scrape(
+  'https://firecrawl.dev', 
+  formats=['markdown', 'html']
+)
+print(data)
+
+# Crawl a website (v2 waiter):
+crawl_status = firecrawl.crawl(
+  'https://firecrawl.dev', 
+  limit=100, 
+  scrape_options=ScrapeOptions(formats=['markdown', 'html'])
+)
+print(crawl_status)
+```
+
+### Scraping a URL
+
+To scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats.
+
+```python 
+# Scrape a website (v2):
+scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', 'html'])
+print(scrape_result)
+```
+
+### Crawling a Website
+
+To crawl a website, use the `crawl` method. It takes the starting URL and optional parameters as arguments. You can control depth, limits, formats, and more.
+
+```python 
+crawl_status = firecrawl.crawl(
+  'https://firecrawl.dev', 
+  limit=100, 
+  scrape_options=ScrapeOptions(formats=['markdown', 'html']),
+  poll_interval=30
+)
+print(crawl_status)
+```
+
+### Asynchronous Crawling
+
+<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>
+
+To enqueue a crawl asynchronously, use `start_crawl`. It returns the crawl `ID` which you can use to check the status of the crawl job.
+
+```python 
+crawl_job = firecrawl.start_crawl(
+  'https://firecrawl.dev', 
+  limit=100, 
+  scrape_options=ScrapeOptions(formats=['markdown', 'html']),
+)
+print(crawl_job)
+```
+
+### Checking Crawl Status
+
+To check the status of a crawl job, use the `get_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
+
+```python 
+crawl_status = firecrawl.get_crawl_status("<crawl_id>")
+print(crawl_status)
+```
+
+### Cancelling a Crawl
+
+To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.
+
+```python 
+cancel_crawl = firecrawl.cancel_crawl(id)
+print(cancel_crawl)
+```
+
+### Map a Website
+
+Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to use the sitemap or include subdomains.
+
+```python 
+# Map a website (v2):
+map_result = firecrawl.map('https://firecrawl.dev')
+print(map_result)
+```
+
+{/* ### Extracting Structured Data from Websites
+
+  To extract structured data from websites, use the `extract` method. It takes the URLs to extract data from, a prompt, and a schema as arguments. The schema is a Pydantic model that defines the structure of the extracted data.
+
+  <ExtractPythonShort /> */}
+
+### Crawling a Website with WebSockets
+
+To crawl a website with WebSockets, use the `crawl_url_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+
+```python 
+# inside an async function...
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+    print("DOC", detail)
+
+def on_error(detail):
+    print("ERR", detail['error'])
+
+def on_done(detail):
+    print("DONE", detail['status'])
+
+    # Function to start the crawl and watch process
+async def start_crawl_and_watch():
+    # Initiate the crawl job and get the watcher
+    watcher = app.crawl_url_and_watch('firecrawl.dev', exclude_paths=['blog/*'], limit=5)
+
+    # Add event listeners
+    watcher.add_event_listener("document", on_document)
+    watcher.add_event_listener("error", on_error)
+    watcher.add_event_listener("done", on_done)
+
+    # Start the watcher
+    await watcher.connect()
+
+# Run the event loop
+await start_crawl_and_watch()
+```
+
+## Error Handling
+
+The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.
+
+## Async Class
+
+For async operations, you can use the `AsyncFirecrawl` class. Its methods mirror the `Firecrawl` class, but you `await` them.
+
+```python 
+from firecrawl import AsyncFirecrawl
+
+firecrawl = AsyncFirecrawl(api_key="YOUR_API_KEY")
+
+# Async Scrape (v2)
+async def example_scrape():
+  scrape_result = await firecrawl.scrape(url="https://example.com")
+  print(scrape_result)
+
+# Async Crawl (v2)
+async def example_crawl():
+  crawl_result = await firecrawl.crawl(url="https://example.com")
+  print(crawl_result)
+```
+
+## v1 compatibility
+
+For legacy code paths, v1 remains available under `firecrawl.v1` with the original method names.
+
+```python
+from firecrawl import Firecrawl
+
+firecrawl = Firecrawl(api_key="YOUR_API_KEY")
+
+# v1 methods (feature‑frozen)
+doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html'])
+crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100)
+map_v1 = firecrawl.v1.map_url('https://firecrawl.dev')
+```

firecrawl_py-4.10.5/README.md

@@ -0,0 +1,187 @@
+# Firecrawl Python SDK
+
+The Firecrawl Python SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
+
+## Installation
+
+To install the Firecrawl Python SDK, you can use pip:
+
+```bash 
+pip install firecrawl-py
+```
+
+## Usage
+
+1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
+2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `Firecrawl` class.
+
+Here's an example of how to use the SDK:
+
+```python 
+from firecrawl import Firecrawl
+from firecrawl.types import ScrapeOptions
+
+firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY")
+
+# Scrape a website (v2):
+data = firecrawl.scrape(
+  'https://firecrawl.dev', 
+  formats=['markdown', 'html']
+)
+print(data)
+
+# Crawl a website (v2 waiter):
+crawl_status = firecrawl.crawl(
+  'https://firecrawl.dev', 
+  limit=100, 
+  scrape_options=ScrapeOptions(formats=['markdown', 'html'])
+)
+print(crawl_status)
+```
+
+### Scraping a URL
+
+To scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats.
+
+```python 
+# Scrape a website (v2):
+scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', 'html'])
+print(scrape_result)
+```
+
+### Crawling a Website
+
+To crawl a website, use the `crawl` method. It takes the starting URL and optional parameters as arguments. You can control depth, limits, formats, and more.
+
+```python 
+crawl_status = firecrawl.crawl(
+  'https://firecrawl.dev', 
+  limit=100, 
+  scrape_options=ScrapeOptions(formats=['markdown', 'html']),
+  poll_interval=30
+)
+print(crawl_status)
+```
+
+### Asynchronous Crawling
+
+<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>
+
+To enqueue a crawl asynchronously, use `start_crawl`. It returns the crawl `ID` which you can use to check the status of the crawl job.
+
+```python 
+crawl_job = firecrawl.start_crawl(
+  'https://firecrawl.dev', 
+  limit=100, 
+  scrape_options=ScrapeOptions(formats=['markdown', 'html']),
+)
+print(crawl_job)
+```
+
+### Checking Crawl Status
+
+To check the status of a crawl job, use the `get_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
+
+```python 
+crawl_status = firecrawl.get_crawl_status("<crawl_id>")
+print(crawl_status)
+```
+
+### Cancelling a Crawl
+
+To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.
+
+```python 
+cancel_crawl = firecrawl.cancel_crawl(id)
+print(cancel_crawl)
+```
+
+### Map a Website
+
+Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to use the sitemap or include subdomains.
+
+```python 
+# Map a website (v2):
+map_result = firecrawl.map('https://firecrawl.dev')
+print(map_result)
+```
+
+{/* ### Extracting Structured Data from Websites
+
+  To extract structured data from websites, use the `extract` method. It takes the URLs to extract data from, a prompt, and a schema as arguments. The schema is a Pydantic model that defines the structure of the extracted data.
+
+  <ExtractPythonShort /> */}
+
+### Crawling a Website with WebSockets
+
+To crawl a website with WebSockets, use the `crawl_url_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+
+```python 
+# inside an async function...
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+    print("DOC", detail)
+
+def on_error(detail):
+    print("ERR", detail['error'])
+
+def on_done(detail):
+    print("DONE", detail['status'])
+
+    # Function to start the crawl and watch process
+async def start_crawl_and_watch():
+    # Initiate the crawl job and get the watcher
+    watcher = app.crawl_url_and_watch('firecrawl.dev', exclude_paths=['blog/*'], limit=5)
+
+    # Add event listeners
+    watcher.add_event_listener("document", on_document)
+    watcher.add_event_listener("error", on_error)
+    watcher.add_event_listener("done", on_done)
+
+    # Start the watcher
+    await watcher.connect()
+
+# Run the event loop
+await start_crawl_and_watch()
+```
+
+## Error Handling
+
+The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.
+
+## Async Class
+
+For async operations, you can use the `AsyncFirecrawl` class. Its methods mirror the `Firecrawl` class, but you `await` them.
+
+```python 
+from firecrawl import AsyncFirecrawl
+
+firecrawl = AsyncFirecrawl(api_key="YOUR_API_KEY")
+
+# Async Scrape (v2)
+async def example_scrape():
+  scrape_result = await firecrawl.scrape(url="https://example.com")
+  print(scrape_result)
+
+# Async Crawl (v2)
+async def example_crawl():
+  crawl_result = await firecrawl.crawl(url="https://example.com")
+  print(crawl_result)
+```
+
+## v1 compatibility
+
+For legacy code paths, v1 remains available under `firecrawl.v1` with the original method names.
+
+```python
+from firecrawl import Firecrawl
+
+firecrawl = Firecrawl(api_key="YOUR_API_KEY")
+
+# v1 methods (feature‑frozen)
+doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html'])
+crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100)
+map_v1 = firecrawl.v1.map_url('https://firecrawl.dev')
+```

firecrawl_py-4.10.5/firecrawl/__init__.py

@@ -0,0 +1,87 @@
+"""
+Firecrawl Python SDK
+
+"""
+
+import logging
+import os
+
+from .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlApp
+from .v2.watcher import Watcher
+from .v2.watcher_async import AsyncWatcher
+from .v1 import (
+    V1FirecrawlApp,
+    AsyncV1FirecrawlApp,
+    V1JsonConfig,
+    V1ScrapeOptions,
+    V1ChangeTrackingOptions,
+)
+
+__version__ = "4.10.5"
+
+# Define the logger for the Firecrawl project
+logger: logging.Logger = logging.getLogger("firecrawl")
+
+
+def _configure_logger() -> None:
+    """
+    Configure the firecrawl logger for console output.
+
+    The function attaches a handler for console output with a specific format and date
+    format to the firecrawl logger.
+    """
+    try:
+        formatter = logging.Formatter(
+            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+
+        logger.addHandler(console_handler)
+    except Exception as e:
+        logger.error("Failed to configure logging: %s", e)
+
+
+def setup_logging() -> None:
+    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
+    if logger.hasHandlers():
+        return
+
+    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
+        logger.addHandler(logging.NullHandler()) 
+        return
+
+    _configure_logger()
+
+    if env == "DEBUG":
+        logger.setLevel(logging.DEBUG)
+    elif env == "INFO":
+        logger.setLevel(logging.INFO)
+    elif env == "WARNING":
+        logger.setLevel(logging.WARNING)
+    elif env == "ERROR":
+        logger.setLevel(logging.ERROR)
+    elif env == "CRITICAL":
+        logger.setLevel(logging.CRITICAL)
+    else:
+        logger.setLevel(logging.INFO)
+        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
+
+setup_logging()
+logger.debug("Debugging logger setup")
+
+__all__ = [
+    'Firecrawl',
+    'AsyncFirecrawl',
+    'FirecrawlApp',
+    'AsyncFirecrawlApp',
+    'Watcher',
+    'AsyncWatcher',
+    'V1FirecrawlApp',
+    'AsyncV1FirecrawlApp',
+    'V1JsonConfig',
+    'V1ScrapeOptions',
+    'V1ChangeTrackingOptions',
+]

firecrawl_py-4.10.5/firecrawl/__tests__/e2e/v2/aio/conftest.py

@@ -0,0 +1,62 @@
+"""
+Pytest configuration for async E2E tests.
+Ensures environment variables are loaded before any test runs.
+"""
+import os
+import pytest
+from dotenv import load_dotenv, find_dotenv
+
+# Load environment IMMEDIATELY at module import time (before pytest collects tests)
+# This ensures env vars are loaded before the first test runs
+_env_loaded = False
+
+def _ensure_env_loaded():
+    """Ensure environment is loaded exactly once, synchronously."""
+    global _env_loaded
+    if not _env_loaded:
+        # Find and load .env from multiple possible locations
+        env_file = find_dotenv(usecwd=True)
+        if env_file:
+            load_dotenv(env_file, override=True)
+        else:
+            # Try loading from current directory
+            load_dotenv(override=True)
+        _env_loaded = True
+
+# Load env immediately when this module is imported
+_ensure_env_loaded()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def load_environment():
+    """Ensure environment variables are loaded before running any tests."""
+    # Double-check env is loaded (should already be done at import time)
+    _ensure_env_loaded()
+    
+    # Validate required environment variables
+    required_vars = ["API_KEY", "API_URL"]
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    
+    if missing_vars:
+        pytest.skip(f"Skipping E2E tests: Missing required environment variables: {', '.join(missing_vars)}")
+    
+    yield
+
+
+@pytest.fixture(scope="function")
+def api_key():
+    """Provide API key for tests."""
+    key = os.getenv("API_KEY")
+    if not key:
+        pytest.skip("API_KEY not set")
+    return key
+
+
+@pytest.fixture(scope="function")
+def api_url():
+    """Provide API URL for tests."""
+    url = os.getenv("API_URL")
+    if not url:
+        pytest.skip("API_URL not set")
+    return url
+

firecrawl_py-4.10.5/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py

@@ -0,0 +1,69 @@
+import asyncio
+import pytest
+from firecrawl import AsyncFirecrawl
+
+
+@pytest.mark.asyncio
+async def test_async_batch_start_and_status(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    job_id = start.id
+
+    deadline = asyncio.get_event_loop().time() + 240
+    status = await client.get_batch_scrape_status(job_id)
+    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
+        await asyncio.sleep(2)
+        status = await client.get_batch_scrape_status(job_id)
+
+    assert status.status in ("completed", "failed", "cancelled")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_wait_minimal(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    job = await client.batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], poll_interval=1, timeout=120)
+    assert job.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_batch_wait_with_all_params(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
+    job = await client.batch_scrape(
+        [
+            "https://docs.firecrawl.dev",
+            "https://firecrawl.dev",
+        ],
+        formats=[
+            "markdown",
+            {"type": "json", "prompt": "Extract page title", "schema": json_schema},
+            {"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
+        ],
+        only_main_content=True,
+        mobile=False,
+        ignore_invalid_urls=True,
+        max_concurrency=2,
+        zero_data_retention=False,
+        poll_interval=1,
+        timeout=180,
+        integration="_e2e-test",
+    )
+    assert job.status in ("completed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_async_cancel_batch(api_key, api_url):
+    client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
+    start = await client.start_batch_scrape([
+        "https://docs.firecrawl.dev",
+        "https://firecrawl.dev",
+    ], formats=["markdown"], max_concurrency=1)
+    ok = await client.cancel_batch_scrape(start.id)
+    assert ok is True
+