firecrawl 2.16.5__py3-none-any.whl → 3.0.3__py3-none-any.whl

firecrawl/v2/methods/scrape.py

@@ -0,0 +1,68 @@
+"""
+Scraping functionality for Firecrawl v2 API.
+"""
+
+from typing import Optional, Dict, Any
+from ..types import ScrapeOptions, Document
+from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
+
+
+def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
+    """
+    Prepare a scrape request payload for v2 API.
+    
+    Args:
+        url: URL to scrape
+        options: ScrapeOptions (snake_case) to convert and include
+        
+    Returns:
+        Request payload dictionary with camelCase fields
+    """
+    if not url or not url.strip():
+        raise ValueError("URL cannot be empty")
+
+    request_data: Dict[str, Any] = {"url": url.strip()}
+
+    if options is not None:
+        validated = validate_scrape_options(options)
+        if validated is not None:
+            opts = prepare_scrape_options(validated)
+            if opts:
+                request_data.update(opts)
+
+    return request_data
+
+def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
+    """
+    Scrape a single URL and return the document.
+    
+    The v2 API returns: { success: boolean, data: Document }
+    We surface just the Document to callers.
+    
+    Args:
+        client: HTTP client instance
+        url: URL to scrape
+        options: Scraping options (snake_case)
+        
+    Returns:
+        Document
+    """
+    payload = _prepare_scrape_request(url, options)
+
+    response = client.post("/v2/scrape", payload)
+
+    if not response.ok:
+        handle_response_error(response, "scrape")
+
+    body = response.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error occurred"))
+
+    document_data = body.get("data", {})
+    # Normalize keys for Document (no Pydantic aliases)
+    normalized = dict(document_data)
+    if 'rawHtml' in normalized and 'raw_html' not in normalized:
+        normalized['raw_html'] = normalized.pop('rawHtml')
+    if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+        normalized['change_tracking'] = normalized.pop('changeTracking')
+    return Document(**normalized)

firecrawl/v2/methods/search.py

@@ -0,0 +1,173 @@
+"""
+Search functionality for Firecrawl v2 API.
+"""
+
+from typing import Optional, Dict, Any, Union
+from ..types import SearchRequest, SearchData, SearchResult, Document
+from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+
+
+def search(
+    client: HttpClient,
+    request: SearchRequest
+) -> SearchData:
+    """
+    Search for documents.
+    
+    Args:
+        client: HTTP client instance
+        request: Search request
+        
+    Returns:
+        SearchData with search results grouped by source type
+        
+    Raises:
+        FirecrawlError: If the search operation fails
+    """
+    request_data = _prepare_search_request(request)
+    
+    response = client.post("/v2/search", request_data)
+    
+    if not response.ok:
+        handle_response_error(response, "search")
+    
+    response_data = response.json()
+    
+    if not response_data.get("success"):
+        # Handle error case
+        error_msg = response_data.get("error", "Unknown error occurred")
+        raise Exception(f"Search failed: {error_msg}")
+    
+    data = response_data.get("data", {})
+    search_data = SearchData()
+    
+    for source_type, source_documents in data.items():
+        if isinstance(source_documents, list):
+            results = []
+            for doc_data in source_documents:
+                if isinstance(doc_data, dict):
+                    # If page scraping options were provided, API returns full Document objects
+                    if request.scrape_options is not None and any(
+                        key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']
+                    ):
+                        # Normalize keys for Document (no Pydantic aliases)
+                        normalized = dict(doc_data)
+                        if 'rawHtml' in normalized and 'raw_html' not in normalized:
+                            normalized['raw_html'] = normalized.pop('rawHtml')
+                        if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+                            normalized['change_tracking'] = normalized.pop('changeTracking')
+                        results.append(Document(**normalized))
+                    else:
+                        # Minimal search result shape
+                        results.append(SearchResult(
+                            url=doc_data.get('url', ''),
+                            title=doc_data.get('title'),
+                            description=doc_data.get('description')
+                        ))
+                elif isinstance(doc_data, str):
+                    results.append(SearchResult(url=doc_data))
+
+            if hasattr(search_data, source_type):
+                setattr(search_data, source_type, results)
+    
+    return search_data
+
+
+def _validate_search_request(request: SearchRequest) -> SearchRequest:
+    """
+    Validate and normalize search request.
+    
+    Args:
+        request: Search request to validate
+        
+    Returns:
+        Validated request
+        
+    Raises:
+        ValueError: If request is invalid
+    """
+    # Validate query
+    if not request.query or not request.query.strip():
+        raise ValueError("Query cannot be empty")
+    
+    # Validate limit
+    if request.limit is not None:
+        if request.limit <= 0:
+            raise ValueError("Limit must be positive")
+        if request.limit > 100:
+            raise ValueError("Limit cannot exceed 100")
+    
+    # Validate timeout
+    if request.timeout is not None:
+        if request.timeout <= 0:
+            raise ValueError("Timeout must be positive")
+        if request.timeout > 300000:  # 5 minutes max
+            raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
+    
+    # Validate sources (if provided)
+    if request.sources is not None:
+        valid_sources = {"web", "news", "images"}
+        for source in request.sources:
+            if isinstance(source, str):
+                if source not in valid_sources:
+                    raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
+            elif hasattr(source, 'type'):
+                if source.type not in valid_sources:
+                    raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
+    
+    # Validate location (if provided)
+    if request.location is not None:
+        if not isinstance(request.location, str) or len(request.location.strip()) == 0:
+            raise ValueError("Location must be a non-empty string")
+    
+    # Validate tbs (time-based search, if provided)
+    if request.tbs is not None:
+        valid_tbs_values = {
+            "qdr:d", "qdr:w", "qdr:m", "qdr:y",  # Google time filters
+            "d", "w", "m", "y"  # Short forms
+        }
+        if request.tbs not in valid_tbs_values:
+            raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values}")
+    
+    # Validate scrape_options (if provided)
+    if request.scrape_options is not None:
+        validate_scrape_options(request.scrape_options)
+    
+    return request
+
+
+def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
+    """
+    Prepare a search request payload.
+    
+    Args:
+        request: Search request
+        
+    Returns:
+        Request payload dictionary
+    """
+    validated_request = _validate_search_request(request)
+    data = validated_request.model_dump(exclude_none=True, by_alias=True)
+    
+    # Ensure default values are included only if not explicitly set to None
+    if "limit" not in data and validated_request.limit is not None:
+        data["limit"] = validated_request.limit
+    if "timeout" not in data and validated_request.timeout is not None:
+        data["timeout"] = validated_request.timeout
+    
+    # Handle snake_case to camelCase conversions manually
+    # (Pydantic Field() aliases interfere with value assignment)
+    
+    # ignore_invalid_urls → ignoreInvalidURLs
+    if validated_request.ignore_invalid_urls is not None:
+        data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
+        data.pop("ignore_invalid_urls", None)
+    
+    # scrape_options → scrapeOptions
+    if validated_request.scrape_options is not None:
+        scrape_data = prepare_scrape_options(validated_request.scrape_options)
+        if scrape_data:
+            data["scrapeOptions"] = scrape_data
+        data.pop("scrape_options", None)
+    
+    return data

firecrawl/v2/methods/usage.py

@@ -0,0 +1,41 @@
+from ..utils import HttpClient, handle_response_error
+from ..types import ConcurrencyCheck, CreditUsage, TokenUsage
+
+
+def get_concurrency(client: HttpClient) -> ConcurrencyCheck:
+    resp = client.get("/v2/concurrency-check")
+    if not resp.ok:
+        handle_response_error(resp, "get concurrency")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return ConcurrencyCheck(
+        concurrency=data.get("concurrency"),
+        max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
+    )
+
+
+def get_credit_usage(client: HttpClient) -> CreditUsage:
+    resp = client.get("/v2/team/credit-usage")
+    if not resp.ok:
+        handle_response_error(resp, "get credit usage")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return CreditUsage(remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)))
+
+
+def get_token_usage(client: HttpClient) -> TokenUsage:
+    resp = client.get("/v2/team/token-usage")
+    if not resp.ok:
+        handle_response_error(resp, "get token usage")
+    body = resp.json()
+    if not body.get("success"):
+        raise Exception(body.get("error", "Unknown error"))
+    data = body.get("data", body)
+    return TokenUsage(
+        remaining_tokens=data.get("remainingTokens", 0)
+    )
+