onetool-mcp 1.0.0b1__py3-none-any.whl

ot_tools/firecrawl.py

@@ -0,0 +1,732 @@
+"""Web scraping, crawling, and structured extraction via Firecrawl API.
+
+Provides single URL scraping, batch scraping, URL discovery, web search,
+multi-page crawling, and LLM-powered data extraction.
+
+API docs: https://docs.firecrawl.dev/api-reference
+Python SDK: https://pypi.org/project/firecrawl/
+"""
+
+from __future__ import annotations
+
+# Pack for dot notation: firecrawl.scrape(), firecrawl.crawl(), etc.
+pack = "firecrawl"
+
+__all__ = [
+    "crawl",
+    "crawl_status",
+    "deep_research",
+    "extract",
+    "map_urls",
+    "scrape",
+    "scrape_batch",
+    "search",
+]
+
+# Dependency declarations for CLI validation
+__ot_requires__ = {
+    "lib": [("firecrawl", "pip install firecrawl")],
+    "secrets": ["FIRECRAWL_API_KEY"],
+}
+
+from typing import Any, Literal
+from urllib.parse import urlparse
+
+from firecrawl import Firecrawl
+from pydantic import BaseModel, Field
+
+from ot.config import get_secret, get_tool_config
+from ot.logging import LogSpan
+from ot.utils import batch_execute, lazy_client, normalize_items
+
+
+def _validate_url(url: str) -> str | None:
+    """Return error message if URL is invalid, None otherwise."""
+    if not url or not url.strip():
+        return "URL is required and cannot be empty"
+    try:
+        parsed = urlparse(url)
+        if not parsed.scheme or not parsed.netloc:
+            return f"Invalid URL: {url} (missing scheme or host)"
+        if parsed.scheme not in ("http", "https"):
+            return f"Invalid URL scheme: {parsed.scheme} (expected http/https)"
+    except Exception as e:
+        return f"Invalid URL: {e}"
+    return None
+
+
+class Config(BaseModel):
+    """Pack configuration - discovered by registry."""
+
+    api_url: str | None = Field(
+        default=None,
+        description="Custom API URL for self-hosted Firecrawl instances",
+    )
+
+
+def _get_config() -> Config:
+    """Get firecrawl pack configuration."""
+    return get_tool_config("firecrawl", Config)
+
+
+def _create_client() -> Firecrawl | None:
+    """Create Firecrawl client with API key."""
+    api_key = get_secret("FIRECRAWL_API_KEY")
+    if not api_key:
+        return None
+
+    api_url = _get_config().api_url
+    if api_url:
+        return Firecrawl(api_key=api_key, api_url=api_url)
+    return Firecrawl(api_key=api_key)
+
+
+# Thread-safe lazy client using SDK utility
+_get_client = lazy_client(_create_client)
+
+
+def _to_dict(obj: Any) -> dict[str, Any]:
+    """Convert SDK response objects to dicts for JSON serialization."""
+    if isinstance(obj, dict):
+        return obj
+    # Pydantic v2
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    # Pydantic v1
+    if hasattr(obj, "dict"):
+        return obj.dict()
+    # Fallback for dataclasses or other objects
+    if hasattr(obj, "__dict__"):
+        return {k: v for k, v in obj.__dict__.items() if not k.startswith("_")}
+    return {"value": str(obj)}
+
+
+def scrape(
+    *,
+    url: str,
+    formats: list[
+        Literal[
+            "markdown", "html", "rawHtml", "links", "screenshot", "screenshot@fullPage"
+        ]
+    ]
+    | None = None,
+    only_main_content: bool = True,
+    include_tags: list[str] | None = None,
+    exclude_tags: list[str] | None = None,
+    wait_for: int | None = None,
+    timeout: int | None = None,
+    mobile: bool = False,
+    skip_tls_verification: bool = False,
+    remove_base64_images: bool = True,
+    location: dict[str, Any] | None = None,
+) -> dict[str, Any] | str:
+    """Scrape content from a single URL.
+
+    Extracts content in various formats with configurable filtering.
+
+    Args:
+        url: The URL to scrape
+        formats: Output formats to include. Options:
+            - "markdown": Clean markdown text (default)
+            - "html": Cleaned HTML
+            - "rawHtml": Original HTML
+            - "links": All hyperlinks on the page
+            - "screenshot": Screenshot image (base64)
+            - "screenshot@fullPage": Full page screenshot
+        only_main_content: Extract only main content, excluding nav/footer (default: True)
+        include_tags: HTML tags to include (e.g., ["article", "main"])
+        exclude_tags: HTML tags to exclude (e.g., ["nav", "footer"])
+        wait_for: Milliseconds to wait for dynamic content
+        timeout: Request timeout in milliseconds
+        mobile: Use mobile user agent
+        skip_tls_verification: Skip TLS certificate validation
+        remove_base64_images: Remove base64 images from markdown (default: True)
+        location: Geolocation for request (e.g., {"country": "US", "languages": ["en"]})
+
+    Returns:
+        Dict with scraped content in requested formats, or error message
+
+    Example:
+        # Basic scrape
+        firecrawl.scrape(url="https://example.com")
+
+        # Get markdown and links
+        firecrawl.scrape(url="https://example.com", formats=["markdown", "links"])
+
+        # Scrape with geolocation
+        firecrawl.scrape(url="https://example.com", location={"country": "US"})
+    """
+    # Validate URL
+    if url_error := _validate_url(url):
+        return f"Error: {url_error}"
+
+    with LogSpan(span="firecrawl.scrape", url=url) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        try:
+            # Build kwargs for v2 API
+            kwargs: dict[str, Any] = {}
+
+            if formats:
+                kwargs["formats"] = formats
+            if not only_main_content:
+                kwargs["only_main_content"] = False
+            if include_tags:
+                kwargs["include_tags"] = include_tags
+            if exclude_tags:
+                kwargs["exclude_tags"] = exclude_tags
+            if wait_for is not None:
+                kwargs["wait_for"] = wait_for
+            if timeout is not None:
+                kwargs["timeout"] = timeout
+            if mobile:
+                kwargs["mobile"] = True
+            if skip_tls_verification:
+                kwargs["skip_tls_verification"] = True
+            if not remove_base64_images:
+                kwargs["remove_base64_images"] = False
+            if location:
+                kwargs["location"] = location
+
+            result = client.scrape(url, **kwargs)
+
+            span.add(success=True)
+            result_dict = _to_dict(result)
+            if isinstance(result_dict, dict):
+                span.add(formats=list(result_dict.keys()))
+            return result_dict
+
+        except Exception as e:
+            error_msg = f"Scrape failed: {e}"
+            span.add(error=str(e))
+            return error_msg
+
+
+def scrape_batch(
+    *,
+    urls: list[str] | list[tuple[str, str]],
+    formats: list[
+        Literal[
+            "markdown", "html", "rawHtml", "links", "screenshot", "screenshot@fullPage"
+        ]
+    ]
+    | None = None,
+    only_main_content: bool = True,
+    max_workers: int = 5,
+) -> dict[str, dict[str, Any] | str]:
+    """Scrape multiple URLs concurrently.
+
+    Uses ThreadPoolExecutor for parallel execution with error isolation.
+
+    Args:
+        urls: List of URLs to scrape. Each item can be:
+            - A string (URL used as key)
+            - A tuple of (url, label) for custom labeling
+        formats: Output formats (see scrape() for options)
+        only_main_content: Extract only main content (default: True)
+        max_workers: Maximum concurrent scrapes (default: 5)
+
+    Returns:
+        Dict mapping URL/label to scraped content or error message
+
+    Example:
+        # Simple list
+        firecrawl.scrape_batch(urls=[
+            "https://docs.python.org/3/library/asyncio.html",
+            "https://docs.python.org/3/library/threading.html",
+        ])
+
+        # With labels
+        firecrawl.scrape_batch(urls=[
+            ("https://example.com/page1", "Page 1"),
+            ("https://example.com/page2", "Page 2"),
+        ])
+    """
+    normalized = normalize_items(urls)
+
+    with LogSpan(span="firecrawl.scrape_batch", url_count=len(normalized)) as span:
+
+        def _scrape_one(url: str, label: str) -> tuple[str, dict[str, Any] | str]:
+            result = scrape(
+                url=url,
+                formats=formats,
+                only_main_content=only_main_content,
+            )
+            return label, result
+
+        results = batch_execute(_scrape_one, normalized, max_workers=max_workers)
+        span.add(success_count=sum(1 for r in results.values() if isinstance(r, dict)))
+        return results
+
+
+def map_urls(
+    *,
+    url: str,
+    search: str | None = None,
+    ignore_sitemap: bool = False,
+    sitemap_only: bool = False,
+    include_subdomains: bool = False,
+    limit: int | None = None,
+) -> list[str] | str:
+    """Discover URLs from a website.
+
+    Maps all accessible URLs from a site via sitemap and crawling.
+
+    Args:
+        url: The starting URL to map
+        search: Optional search term to filter URLs
+        ignore_sitemap: Skip sitemap discovery, only crawl (default: False)
+        sitemap_only: Only use sitemap, no crawling (default: False)
+        include_subdomains: Include URLs from subdomains (default: False)
+        limit: Maximum number of URLs to return
+
+    Returns:
+        List of discovered URLs, or error message
+
+    Example:
+        # Map entire site
+        firecrawl.map_urls(url="https://docs.python.org")
+
+        # Search for specific pages
+        firecrawl.map_urls(url="https://docs.python.org", search="asyncio")
+
+        # Limit results
+        firecrawl.map_urls(url="https://example.com", limit=100)
+    """
+    # Validate URL
+    if url_error := _validate_url(url):
+        return f"Error: {url_error}"
+
+    with LogSpan(span="firecrawl.map_urls", url=url, search=search) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        try:
+            # Build kwargs for v2 API
+            kwargs: dict[str, Any] = {}
+
+            if search:
+                kwargs["search"] = search
+            if ignore_sitemap:
+                kwargs["sitemap"] = "skip"
+            if sitemap_only:
+                kwargs["sitemap"] = "only"
+            if include_subdomains:
+                kwargs["include_subdomains"] = True
+            if limit:
+                kwargs["limit"] = limit
+
+            result = client.map(url, **kwargs)
+
+            # Extract links list from result
+            if isinstance(result, list):
+                links = result
+            else:
+                # Handle MapResponse object
+                links = getattr(result, "links", None) or []
+
+            # Convert LinkResult objects to URL strings
+            urls = []
+            for link in links:
+                if isinstance(link, str):
+                    urls.append(link)
+                elif hasattr(link, "url"):
+                    urls.append(link.url)
+                elif hasattr(link, "model_dump"):
+                    urls.append(link.model_dump().get("url", str(link)))
+                else:
+                    urls.append(str(link))
+
+            span.add(url_count=len(urls))
+            return urls
+
+        except Exception as e:
+            error_msg = f"Map failed: {e}"
+            span.add(error=str(e))
+            return error_msg
+
+
+def search(
+    *,
+    query: str,
+    limit: int = 5,
+    lang: str | None = None,
+    country: str | None = None,
+    scrape_options: dict[str, Any] | None = None,
+) -> list[dict[str, Any]] | str:
+    """Search the web and optionally scrape results.
+
+    Performs web search with optional content retrieval for each result.
+
+    Args:
+        query: Search query string
+        limit: Maximum number of results (default: 5)
+        lang: Language code for results (e.g., "en")
+        country: Country code for results (e.g., "US")
+        scrape_options: Options for scraping result pages (see scrape() params)
+
+    Returns:
+        List of search results with optional scraped content, or error message
+
+    Example:
+        # Basic search
+        firecrawl.search(query="Python async best practices")
+
+        # Search with scraping
+        firecrawl.search(
+            query="machine learning tutorials",
+            limit=3,
+            scrape_options={"formats": ["markdown"]}
+        )
+    """
+    with LogSpan(span="firecrawl.search", query=query, limit=limit) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        try:
+            # Build kwargs for v2 API
+            kwargs: dict[str, Any] = {"limit": limit}
+
+            if lang:
+                kwargs["lang"] = lang
+            if country:
+                kwargs["location"] = country
+            if scrape_options:
+                kwargs["scrape_options"] = scrape_options
+
+            result = client.search(query, **kwargs)
+
+            if isinstance(result, list):
+                span.add(result_count=len(result))
+                return [_to_dict(item) for item in result]
+            # Handle SearchData object (v2 API returns .web, .news, .images)
+            data = getattr(result, "web", None) or getattr(result, "data", None) or []
+            span.add(result_count=len(data))
+            return [_to_dict(item) for item in data]
+
+        except Exception as e:
+            error_msg = f"Search failed: {e}"
+            span.add(error=str(e))
+            return error_msg
+
+
+def crawl(
+    *,
+    url: str,
+    max_depth: int | None = None,
+    limit: int | None = None,
+    include_paths: list[str] | None = None,
+    exclude_paths: list[str] | None = None,
+    ignore_sitemap: bool = False,
+    scrape_options: dict[str, Any] | None = None,
+    webhook: str | None = None,
+) -> dict[str, Any] | str:
+    """Start an asynchronous multi-page crawl job.
+
+    Crawls a website starting from the given URL. Returns immediately with
+    a job ID. Use crawl_status() to poll for results.
+
+    Args:
+        url: The starting URL to crawl
+        max_depth: Maximum link depth to crawl
+        limit: Maximum number of pages to crawl
+        include_paths: URL patterns to include (glob syntax)
+        exclude_paths: URL patterns to exclude (glob syntax)
+        ignore_sitemap: Skip sitemap discovery (default: False)
+        scrape_options: Options for scraping pages (see scrape() params)
+        webhook: URL to receive completion notification
+
+    Returns:
+        Dict with job ID and status URL, or error message
+
+    Example:
+        # Start a crawl
+        job = firecrawl.crawl(url="https://docs.python.org", max_depth=2, limit=100)
+
+        # Check status
+        firecrawl.crawl_status(id=job["id"])
+    """
+    # Validate URL
+    if url_error := _validate_url(url):
+        return f"Error: {url_error}"
+
+    with LogSpan(span="firecrawl.crawl", url=url, max_depth=max_depth, limit=limit) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        try:
+            # Build kwargs for v2 API
+            kwargs: dict[str, Any] = {}
+
+            if max_depth is not None:
+                kwargs["max_discovery_depth"] = max_depth
+            if limit is not None:
+                kwargs["limit"] = limit
+            if include_paths:
+                kwargs["include_paths"] = include_paths
+            if exclude_paths:
+                kwargs["exclude_paths"] = exclude_paths
+            if ignore_sitemap:
+                kwargs["ignore_sitemap"] = True
+            if scrape_options:
+                kwargs["scrape_options"] = scrape_options
+            if webhook:
+                kwargs["webhook"] = webhook
+
+            result = client.crawl(url, **kwargs)
+
+            # Convert to dict for consistent handling
+            result_dict = _to_dict(result)
+
+            # Extract job ID with multiple fallbacks
+            job_id = (
+                result_dict.get("id")
+                or result_dict.get("jobId")
+                or result_dict.get("job_id")
+            )
+
+            span.add(job_id=job_id)
+
+            # If result already has data (sync completion), return as-is
+            if result_dict.get("data"):
+                return result_dict
+
+            # Return normalized response with job info
+            return {
+                "id": job_id,
+                "status": result_dict.get("status", "started"),
+                "url": url,
+            }
+
+        except Exception as e:
+            error_msg = f"Crawl failed: {e}"
+            span.add(error=str(e))
+            return error_msg
+
+
+def crawl_status(
+    *,
+    id: str,
+) -> dict[str, Any] | str:
+    """Check the status of a crawl job.
+
+    Polls the crawl job for current progress and results.
+
+    Args:
+        id: The crawl job ID returned by crawl()
+
+    Returns:
+        Dict with status, progress, and results (if complete), or error message
+
+    Example:
+        # Check crawl progress
+        status = firecrawl.crawl_status(id="abc123")
+
+        if status["status"] == "completed":
+            for page in status["data"]:
+                print(page["url"])
+    """
+    # Validate job ID
+    if not id or not id.strip():
+        return "Error: Job ID is required and cannot be empty"
+
+    with LogSpan(span="firecrawl.crawl_status", job_id=id) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        try:
+            result = client.get_crawl_status(id)
+
+            if isinstance(result, dict):
+                span.add(status=result.get("status"))
+                return result
+
+            # Handle CrawlStatusResponse object
+            status = getattr(result, "status", "unknown")
+            span.add(status=status)
+
+            response: dict[str, Any] = {
+                "id": id,
+                "status": status,
+            }
+
+            # Add optional fields if present
+            if hasattr(result, "completed"):
+                response["completed"] = result.completed
+            if hasattr(result, "total"):
+                response["total"] = result.total
+            if hasattr(result, "data"):
+                response["data"] = result.data
+
+            return response
+
+        except Exception as e:
+            error_msg = f"Status check failed: {e}"
+            span.add(error=str(e))
+            return error_msg
+
+
+def extract(
+    *,
+    urls: list[str],
+    prompt: str | None = None,
+    schema: dict[str, Any] | None = None,
+    system_prompt: str | None = None,
+    allow_external_links: bool = False,
+) -> dict[str, Any] | str:
+    """Extract structured data from URLs using LLM.
+
+    Uses an LLM to extract data matching a JSON schema from web pages.
+
+    Args:
+        urls: URLs to extract data from
+        prompt: Natural language description of what to extract
+        schema: JSON schema defining the structure of extracted data
+            (OpenAI JSON schema format)
+        system_prompt: Custom system prompt for the LLM
+        allow_external_links: Follow external links during extraction (default: False)
+
+    Returns:
+        Dict with extracted data matching schema, or error message
+
+    Example:
+        # Extract with prompt
+        firecrawl.extract(
+            urls=["https://example.com/products"],
+            prompt="Extract product names and prices"
+        )
+
+        # Extract with schema
+        firecrawl.extract(
+            urls=["https://example.com/team"],
+            schema={
+                "type": "object",
+                "properties": {
+                    "team_members": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {"type": "string"},
+                                "role": {"type": "string"}
+                            }
+                        }
+                    }
+                }
+            }
+        )
+    """
+    with LogSpan(span="firecrawl.extract", url_count=len(urls)) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        if not prompt and not schema:
+            return "Error: Either prompt or schema is required"
+
+        try:
+            # Build kwargs for v2 API
+            kwargs: dict[str, Any] = {}
+
+            if prompt:
+                kwargs["prompt"] = prompt
+            if schema:
+                kwargs["schema"] = schema
+            if system_prompt:
+                kwargs["system_prompt"] = system_prompt
+            if allow_external_links:
+                kwargs["allow_external_links"] = True
+
+            result = client.extract(urls, **kwargs)
+
+            if isinstance(result, dict):
+                span.add(success=True)
+                return result
+
+            # Handle ExtractResponse object
+            data = getattr(result, "data", None)
+            span.add(success=True)
+            return {"data": data} if data else result
+
+        except Exception as e:
+            error_msg = f"Extract failed: {e}"
+            span.add(error=str(e))
+            return error_msg
+
+
+def deep_research(
+    *,
+    prompt: str,
+    urls: list[str] | None = None,
+    timeout: int | None = None,
+    max_credits: int | None = None,
+) -> dict[str, Any] | str:
+    """Run autonomous deep research on a topic.
+
+    Launches an AI agent that autonomously researches a topic by
+    searching, crawling, and synthesizing information from the web.
+
+    Args:
+        prompt: Research question or topic
+        urls: Starting URLs to research (optional, will search if not provided)
+        timeout: Time limit in seconds for the research
+        max_credits: Maximum credits to spend on research
+
+    Returns:
+        Dict with research results and sources, or error message
+
+    Example:
+        # Research a topic
+        firecrawl.deep_research(
+            prompt="What are the latest developments in quantum computing?",
+            timeout=300
+        )
+
+        # Research from specific sources
+        firecrawl.deep_research(
+            prompt="Compare pricing models",
+            urls=["https://company1.com/pricing", "https://company2.com/pricing"]
+        )
+    """
+    with LogSpan(span="firecrawl.deep_research", prompt=prompt[:100]) as span:
+        client = _get_client()
+        if client is None:
+            return "Error: FIRECRAWL_API_KEY secret not configured"
+
+        try:
+            # Build kwargs for v2 API (uses 'agent' method)
+            kwargs: dict[str, Any] = {"prompt": prompt}
+
+            if urls:
+                kwargs["urls"] = urls
+            if timeout is not None:
+                kwargs["timeout"] = timeout
+            if max_credits is not None:
+                kwargs["max_credits"] = max_credits
+
+            # The SDK's agent method corresponds to deep research
+            result = client.agent(**kwargs)
+
+            if isinstance(result, dict):
+                span.add(success=True)
+                return result
+
+            # Handle response object
+            data = getattr(result, "data", None)
+            sources = getattr(result, "sources", None)
+            span.add(success=True, source_count=len(sources) if sources else 0)
+            return {
+                "data": data,
+                "sources": sources,
+            }
+
+        except Exception as e:
+            error_msg = f"Deep research failed: {e}"
+            span.add(error=str(e))
+            return error_msg