firecrawl-py 3.3.1__py3-none-any.whl → 3.3.3__py3-none-any.whl

build/lib/firecrawl/v2/utils/validation.py

@@ -1,324 +0,0 @@
-"""
-Shared validation functions for Firecrawl v2 API.
-"""
-
-from typing import Optional, Dict, Any, List
-from ..types import ScrapeOptions, ScrapeFormats
-
-
-def _convert_format_string(format_str: str) -> str:
-    """
-    Convert format string from snake_case to camelCase.
-    
-    Args:
-        format_str: Format string in snake_case
-        
-    Returns:
-        Format string in camelCase
-    """
-    format_mapping = {
-        "raw_html": "rawHtml",
-        "change_tracking": "changeTracking",
-        "screenshot_full_page": "screenshot@fullPage"
-    }
-    return format_mapping.get(format_str, format_str)
-
-
-def _normalize_schema(schema: Any) -> Optional[Dict[str, Any]]:
-    """
-    Normalize a schema object which may be a dict, Pydantic BaseModel subclass,
-    or a Pydantic model instance into a plain dict.
-    """
-    try:
-        # Pydantic v2 BaseModel subclass: has "model_json_schema"
-        if hasattr(schema, "model_json_schema") and callable(schema.model_json_schema):
-            return schema.model_json_schema()
-        # Pydantic v2 BaseModel instance: has "model_dump" or "model_json_schema"
-        if hasattr(schema, "model_dump") and callable(schema.model_dump):
-            # Try to get JSON schema if available on the class
-            mjs = getattr(schema.__class__, "model_json_schema", None)
-            if callable(mjs):
-                return schema.__class__.model_json_schema()
-            # Fallback to data shape (not ideal, but better than dropping)
-            return schema.model_dump()
-        # Pydantic v1 BaseModel subclass: has "schema"
-        if hasattr(schema, "schema") and callable(schema.schema):
-            return schema.schema()
-        # Pydantic v1 BaseModel instance
-        if hasattr(schema, "dict") and callable(schema.dict):
-            # Prefer class-level schema if present
-            sch = getattr(schema.__class__, "schema", None)
-            if callable(sch):
-                return schema.__class__.schema()
-            return schema.dict()
-    except Exception:
-        pass
-    # Already a dict or unsupported type
-    return schema if isinstance(schema, dict) else None
-
-
-def _validate_json_format(format_obj: Any) -> Dict[str, Any]:
-    """
-    Validate and prepare json format object.
-    
-    Args:
-        format_obj: Format object that should be json type
-        
-    Returns:
-        Validated json format dict
-        
-    Raises:
-        ValueError: If json format is missing required fields
-    """
-    if not isinstance(format_obj, dict):
-        raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
-    
-    if format_obj.get('type') != 'json':
-        raise ValueError("json format must have type='json'")
-    
-    # prompt is optional in v2; only normalize when present
-    # schema is recommended; if provided, normalize Pydantic forms
-    schema = format_obj.get('schema')
-    normalized = dict(format_obj)
-    if schema is not None:
-        normalized_schema = _normalize_schema(schema)
-        if normalized_schema is not None:
-            normalized['schema'] = normalized_schema
-    return normalized
-
-
-def validate_scrape_options(options: Optional[ScrapeOptions]) -> Optional[ScrapeOptions]:
-    """
-    Validate and normalize scrape options.
-    
-    Args:
-        options: Scraping options to validate
-        
-    Returns:
-        Validated options or None
-        
-    Raises:
-        ValueError: If options are invalid
-    """
-    if options is None:
-        return None
-    
-    # Validate timeout
-    if options.timeout is not None and options.timeout <= 0:
-        raise ValueError("Timeout must be positive")
-    
-    # Validate wait_for
-    if options.wait_for is not None and options.wait_for < 0:
-        raise ValueError("wait_for must be non-negative")
-    
-    return options
-
-
-def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[str, Any]]:
-    """
-    Prepare ScrapeOptions for API submission with manual snake_case to camelCase conversion.
-    
-    Args:
-        options: ScrapeOptions to prepare
-        
-    Returns:
-        Dictionary ready for API submission or None if options is None
-    """
-    if options is None:
-        return None
-    
-    # Validate options first
-    validated_options = validate_scrape_options(options)
-    if validated_options is None:
-        return None
-    
-    # Apply default values for None fields
-    default_values = {
-        "only_main_content": True,
-        "mobile": False,
-        "skip_tls_verification": True,
-        "remove_base64_images": True,
-        "fast_mode": False,
-        "block_ads": True,
-        "max_age": 14400000,
-        "store_in_cache": True
-    }
-    
-    # Convert to dict and handle manual snake_case to camelCase conversion
-    options_data = validated_options.model_dump(exclude_none=True)
-    
-    # Apply defaults for None fields
-    for field, default_value in default_values.items():
-        if field not in options_data:
-            options_data[field] = default_value
-    
-    scrape_data = {}
-    
-    # Manual field mapping for snake_case to camelCase conversion
-    field_mappings = {
-        "include_tags": "includeTags",
-        "exclude_tags": "excludeTags",
-        "only_main_content": "onlyMainContent",
-        "wait_for": "waitFor",
-        "skip_tls_verification": "skipTlsVerification",
-        "remove_base64_images": "removeBase64Images",
-        "fast_mode": "fastMode",
-        "use_mock": "useMock",
-        "block_ads": "blockAds",
-        "store_in_cache": "storeInCache",
-        "max_age": "maxAge"
-    }
-    
-    # Apply field mappings
-    for snake_case, camel_case in field_mappings.items():
-        if snake_case in options_data:
-            scrape_data[camel_case] = options_data.pop(snake_case)
-    
-    # Handle special cases
-    for key, value in options_data.items():
-        if value is not None:
-            if key == "formats":
-                # Handle formats conversion
-                converted_formats: List[Any] = []
-
-                # Prefer using original object to detect ScrapeFormats vs list
-                original_formats = getattr(options, 'formats', None)
-
-                if isinstance(original_formats, ScrapeFormats):
-                    # Include explicit list first
-                    if original_formats.formats:
-                        for fmt in original_formats.formats:
-                            if isinstance(fmt, str):
-                                if fmt == "json":
-                                    raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
-                                converted_formats.append(_convert_format_string(fmt))
-                            elif isinstance(fmt, dict):
-                                fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
-                                if fmt_type == 'json':
-                                    validated_json = _validate_json_format({**fmt, 'type': 'json'})
-                                    converted_formats.append(validated_json)
-                                elif fmt_type == 'screenshot':
-                                    # Normalize screenshot options
-                                    normalized = {**fmt, 'type': 'screenshot'}
-                                    if 'full_page' in normalized:
-                                        normalized['fullPage'] = normalized.pop('full_page')
-                                    # Normalize viewport if it's a model instance
-                                    vp = normalized.get('viewport')
-                                    if hasattr(vp, 'model_dump'):
-                                        normalized['viewport'] = vp.model_dump(exclude_none=True)
-                                    converted_formats.append(normalized)
-                                else:
-                                    if 'type' in fmt:
-                                        fmt['type'] = fmt_type or fmt['type']
-                                    converted_formats.append(fmt)
-                            elif hasattr(fmt, 'type'):
-                                if fmt.type == 'json':
-                                    converted_formats.append(_validate_json_format(fmt.model_dump()))
-                                else:
-                                    converted_formats.append(_convert_format_string(fmt.type))
-                            else:
-                                converted_formats.append(fmt)
-
-                    # Add booleans from ScrapeFormats
-                    if original_formats.markdown:
-                        converted_formats.append("markdown")
-                    if original_formats.html:
-                        converted_formats.append("html")
-                    if original_formats.raw_html:
-                        converted_formats.append("rawHtml")
-                    if original_formats.summary:
-                        converted_formats.append("summary")
-                    if original_formats.links:
-                        converted_formats.append("links")
-                    if original_formats.screenshot:
-                        converted_formats.append("screenshot")
-                    if original_formats.change_tracking:
-                        converted_formats.append("changeTracking")
-                    # Note: We intentionally do not auto-include 'json' when boolean is set,
-                    # because JSON requires an object with schema/prompt. The caller must
-                    # supply the full json format object explicitly.
-                elif isinstance(original_formats, list):
-                    for fmt in original_formats:
-                        if isinstance(fmt, str):
-                            if fmt == "json":
-                                raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
-                            converted_formats.append(_convert_format_string(fmt))
-                        elif isinstance(fmt, dict):
-                            fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
-                            if fmt_type == 'json':
-                                validated_json = _validate_json_format({**fmt, 'type': 'json'})
-                                converted_formats.append(validated_json)
-                            elif fmt_type == 'screenshot':
-                                normalized = {**fmt, 'type': 'screenshot'}
-                                if 'full_page' in normalized:
-                                    normalized['fullPage'] = normalized.pop('full_page')
-                                vp = normalized.get('viewport')
-                                if hasattr(vp, 'model_dump'):
-                                    normalized['viewport'] = vp.model_dump(exclude_none=True)
-                                converted_formats.append(normalized)
-                            else:
-                                if 'type' in fmt:
-                                    fmt['type'] = fmt_type or fmt['type']
-                                converted_formats.append(fmt)
-                        elif hasattr(fmt, 'type'):
-                            if fmt.type == 'json':
-                                converted_formats.append(_validate_json_format(fmt.model_dump()))
-                            elif fmt.type == 'screenshot':
-                                normalized = {'type': 'screenshot'}
-                                if getattr(fmt, 'full_page', None) is not None:
-                                    normalized['fullPage'] = fmt.full_page
-                                if getattr(fmt, 'quality', None) is not None:
-                                    normalized['quality'] = fmt.quality
-                                vp = getattr(fmt, 'viewport', None)
-                                if vp is not None:
-                                    normalized['viewport'] = vp.model_dump(exclude_none=True) if hasattr(vp, 'model_dump') else vp
-                                converted_formats.append(normalized)
-                            else:
-                                converted_formats.append(_convert_format_string(fmt.type))
-                        else:
-                            converted_formats.append(fmt)
-                else:
-                    # Fallback: try to iterate over value if it's a list-like
-                    try:
-                        for fmt in value:
-                            converted_formats.append(fmt)
-                    except TypeError:
-                        pass
-
-                if converted_formats:
-                    scrape_data["formats"] = converted_formats
-            elif key == "actions":
-                # Handle actions conversion
-                converted_actions = []
-                for action in value:
-                    if isinstance(action, dict):
-                        # Convert action dict
-                        converted_action = {}
-                        for action_key, action_value in action.items():
-                            if action_key == "full_page":
-                                converted_action["fullPage"] = action_value
-                            else:
-                                converted_action[action_key] = action_value
-                        converted_actions.append(converted_action)
-                    else:
-                        # Handle action objects
-                        action_data = action.model_dump(exclude_none=True)
-                        converted_action = {}
-                        for action_key, action_value in action_data.items():
-                            if action_key == "full_page":
-                                converted_action["fullPage"] = action_value
-                            else:
-                                converted_action[action_key] = action_value
-                        converted_actions.append(converted_action)
-                scrape_data["actions"] = converted_actions
-            elif key == "location":
-                # Handle location conversion
-                if isinstance(value, dict):
-                    scrape_data["location"] = value
-                else:
-                    scrape_data["location"] = value.model_dump(exclude_none=True)
-            else:
-                # For fields that don't need conversion, use as-is
-                scrape_data[key] = value
-    
-    return scrape_data 

build/lib/firecrawl/v2/watcher.py

@@ -1,301 +0,0 @@
-"""
-WebSocket-based watcher for v2 jobs (crawl and batch), mirroring v1 behavior.
-
-Usage:
-    watcher = client.watcher(job_id, kind="crawl")
-    watcher.add_listener(lambda status: print(status.status))
-    watcher.start()
-"""
-
-import asyncio
-import json
-import threading
-from typing import Callable, List, Optional, Literal, Union, Dict, Any
-
-import websockets
-
-from .types import CrawlJob, BatchScrapeJob, Document
-from .utils.normalize import normalize_document_input
-
-
-JobKind = Literal["crawl", "batch"]
-JobType = Union[CrawlJob, BatchScrapeJob]
-
-
-class Watcher:
-    def __init__(
-        self,
-        client: object,
-        job_id: str,
-        kind: JobKind = "crawl",
-        poll_interval: int = 2,
-        timeout: Optional[int] = None,
-    ) -> None:
-        self._client = client
-        self._job_id = job_id
-        self._kind = kind
-        self._timeout = timeout
-        self._poll_interval = poll_interval
-        self._listeners: List[Callable[[JobType], None]] = []
-        self._thread: Optional[threading.Thread] = None
-        self._stop = threading.Event()
-
-        http_client = getattr(client, "http_client", None)
-        self._api_url: Optional[str] = getattr(http_client, "api_url", None)
-        self._api_key: Optional[str] = getattr(http_client, "api_key", None)
-
-        # v1-parity state and event handlers
-        self.status: str = "scraping"
-        self.data: List[Dict[str, Any]] = []
-        self._event_handlers: Dict[str, List[Callable[[Dict[str, Any]], None]]] = {
-            "done": [],
-            "error": [],
-            "document": [],
-        }
-        self._sent_done: bool = False
-        self._sent_error: bool = False
-
-    def add_listener(self, callback: Callable[[JobType], None]) -> None:
-        self._listeners.append(callback)
-
-    def _emit(self, status: JobType) -> None:
-        for cb in list(self._listeners):
-            try:
-                cb(status)
-            except Exception:
-                pass
-
-    # v1-like events API
-    def add_event_listener(self, event_type: str, handler: Callable[[Dict[str, Any]], None]) -> None:
-        if event_type in self._event_handlers:
-            self._event_handlers[event_type].append(handler)
-
-    def dispatch_event(self, event_type: str, detail: Dict[str, Any]) -> None:
-        if event_type in self._event_handlers:
-            for handler in self._event_handlers[event_type]:
-                try:
-                    handler(detail)
-                except Exception:
-                    pass
-
-    def _build_ws_url(self) -> str:
-        if not self._api_url:
-            raise ValueError("API URL is required for WebSocket watcher")
-        ws_base = self._api_url.replace("https://", "wss://").replace("http://", "ws://", 1)
-        if self._kind == "crawl":
-            return f"{ws_base}/v2/crawl/{self._job_id}"
-        return f"{ws_base}/v2/batch/scrape/{self._job_id}"
-
-    async def _run_ws(self) -> None:
-        uri = self._build_ws_url()
-        headers_list = []
-        if self._api_key:
-            headers_list.append(("Authorization", f"Bearer {self._api_key}"))
-
-        try:
-            async with websockets.connect(uri, max_size=None, additional_headers=headers_list) as websocket:
-                deadline = asyncio.get_event_loop().time() + self._timeout if self._timeout else None
-                while not self._stop.is_set():
-                    # Use short recv timeouts to allow HTTP polling fallback
-                    if deadline is not None:
-                        remaining = max(0.0, deadline - asyncio.get_event_loop().time())
-                        timeout = min(self._poll_interval or remaining, remaining)
-                    else:
-                        timeout = self._poll_interval or 5
-                    try:
-                        msg = await asyncio.wait_for(websocket.recv(), timeout=timeout)
-                    except asyncio.TimeoutError:
-                        # Quiet period: poll HTTP once to progress statuses
-                        if await self._poll_status_once():
-                            break
-                        else:
-                            continue
-                    except asyncio.CancelledError:
-                        break
-                    except Exception:
-                        # Connection error: switch to HTTP polling until terminal or timeout
-                        while not self._stop.is_set():
-                            if await self._poll_status_once():
-                                return
-                            if deadline is not None and asyncio.get_event_loop().time() >= deadline:
-                                return
-                            await asyncio.sleep(self._poll_interval or 2)
-                        return
-
-                    try:
-                        body = json.loads(msg)
-                    except Exception:
-                        continue
-
-                    # v1-style typed event handling
-                    msg_type = body.get("type")
-                    if msg_type == "error":
-                        self.status = "failed"
-                        self.dispatch_event("error", {
-                            "status": self.status,
-                            "data": self.data,
-                            "error": body.get("error"),
-                            "id": self._job_id,
-                        })
-                        self._sent_error = True
-                        # Emit a final failed snapshot for listeners
-                        if self._kind == "crawl":
-                            job = CrawlJob(status="failed", completed=0, total=0, credits_used=0, expires_at=None, next=None, data=[])
-                        else:
-                            job = BatchScrapeJob(status="failed", completed=0, total=0, credits_used=0, expires_at=None, next=None, data=[])
-                        self._emit(job)
-                        break
-                    elif msg_type == "catchup":
-                        d = body.get("data", {})
-                        self.status = d.get("status", self.status)
-                        docs_in = d.get("data", [])
-                        self.data.extend(docs_in)
-                        for doc in docs_in:
-                            self.dispatch_event("document", {"data": doc, "id": self._job_id})
-                    elif msg_type == "document":
-                        doc = body.get("data")
-                        if isinstance(doc, dict):
-                            self.data.append(doc)
-                            self.dispatch_event("document", {"data": doc, "id": self._job_id})
-                    elif msg_type == "done":
-                        self.status = "completed"
-                        # Gather any documents in the done payload
-                        raw_payload = body.get("data", {}) or {}
-                        docs_in = raw_payload.get("data", []) or []
-                        if isinstance(docs_in, list) and docs_in:
-                            for doc in docs_in:
-                                if isinstance(doc, dict):
-                                    self.data.append(doc)
-                        # Dispatch done event first
-                        self.dispatch_event("done", {"status": self.status, "data": self.data, "id": self._job_id})
-                        self._sent_done = True
-                        # Emit a final completed snapshot for listeners and break immediately
-                        docs: List[Document] = []
-                        for doc in self.data:
-                            if isinstance(doc, dict):
-                                d = normalize_document_input(doc)
-                                docs.append(Document(**d))
-                        if self._kind == "crawl":
-                            job = CrawlJob(
-                                status="completed",
-                                completed=raw_payload.get("completed", 0),
-                                total=raw_payload.get("total", 0),
-                                credits_used=raw_payload.get("creditsUsed", 0),
-                                expires_at=raw_payload.get("expiresAt"),
-                                next=raw_payload.get("next"),
-                                data=docs,
-                            )
-                        else:
-                            job = BatchScrapeJob(
-                                status="completed",
-                                completed=raw_payload.get("completed", 0),
-                                total=raw_payload.get("total", 0),
-                                credits_used=raw_payload.get("creditsUsed", 0),
-                                expires_at=raw_payload.get("expiresAt"),
-                                next=raw_payload.get("next"),
-                                data=docs,
-                            )
-                        self._emit(job)
-                        break
-
-                    payload = body.get("data", body)
-                    # Only treat messages with an explicit status as job snapshots
-                    has_status_field = (isinstance(payload, dict) and "status" in payload) or ("status" in body)
-                    if not has_status_field:
-                        continue
-                    status_str = payload.get("status", body.get("status", self.status))
-
-                    if self._kind == "crawl":
-                        docs = []
-                        for doc in payload.get("data", []):
-                            if isinstance(doc, dict):
-                                d = normalize_document_input(doc)
-                                docs.append(Document(**d))
-                        job = CrawlJob(
-                            status=status_str,
-                            completed=payload.get("completed", 0),
-                            total=payload.get("total", 0),
-                            credits_used=payload.get("creditsUsed", 0),
-                            expires_at=payload.get("expiresAt"),
-                            next=payload.get("next"),
-                            data=docs,
-                        )
-                        self._emit(job)
-                        if status_str in ("completed", "failed", "cancelled"):
-                            # Ensure done/error dispatched even if server didn't send explicit event type
-                            if status_str == "completed" and not self._sent_done:
-                                self.dispatch_event("done", {"status": status_str, "data": self.data, "id": self._job_id})
-                                self._sent_done = True
-                            if status_str == "failed" and not self._sent_error:
-                                self.dispatch_event("error", {"status": status_str, "data": self.data, "id": self._job_id})
-                                self._sent_error = True
-                            break
-                    else:
-                        docs = []
-                        for doc in payload.get("data", []):
-                            if isinstance(doc, dict):
-                                d = normalize_document_input(doc)
-                                docs.append(Document(**d))
-                        job = BatchScrapeJob(
-                            status=status_str,
-                            completed=payload.get("completed", 0),
-                            total=payload.get("total", 0),
-                            credits_used=payload.get("creditsUsed"),
-                            expires_at=payload.get("expiresAt"),
-                            next=payload.get("next"),
-                            data=docs,
-                        )
-                        self._emit(job)
-                        if status_str in ("completed", "failed", "cancelled"):
-                            if status_str == "completed" and not self._sent_done:
-                                self.dispatch_event("done", {"status": status_str, "data": self.data, "id": self._job_id})
-                                self._sent_done = True
-                            if status_str == "failed" and not self._sent_error:
-                                self.dispatch_event("error", {"status": status_str, "data": self.data, "id": self._job_id})
-                                self._sent_error = True
-                            break
-        except Exception:
-            pass
-        finally:
-            # Ensure terminal event parity with v1 even on abrupt disconnects
-            if self.status == "completed" and not self._sent_done:
-                self.dispatch_event("done", {"status": self.status, "data": self.data, "id": self._job_id})
-                self._sent_done = True
-
-    async def _poll_status_once(self) -> bool:
-        """Poll job status over HTTP once. Returns True if terminal."""
-        try:
-            if self._kind == "crawl":
-                job: CrawlJob = await asyncio.to_thread(self._client.get_crawl_status, self._job_id)
-            else:
-                job: BatchScrapeJob = await asyncio.to_thread(self._client.get_batch_scrape_status, self._job_id)
-        except Exception:
-            return False
-
-        self.status = job.status
-        self._emit(job)
-        if job.status in ("completed", "failed", "cancelled"):
-            if job.status == "completed" and not self._sent_done:
-                self.dispatch_event("done", {"status": job.status, "data": [d.model_dump() for d in job.data], "id": self._job_id})
-                self._sent_done = True
-            if job.status == "failed" and not self._sent_error:
-                self.dispatch_event("error", {"status": job.status, "data": [d.model_dump() for d in job.data], "id": self._job_id})
-                self._sent_error = True
-            return True
-        return False
-
-    def _loop(self) -> None:
-        asyncio.run(self._run_ws())
-
-    def start(self) -> None:
-        if self._thread and self._thread.is_alive():
-            return
-        self._stop.clear()
-        self._thread = threading.Thread(target=self._loop, daemon=True)
-        self._thread.start()
-
-    def stop(self) -> None:
-        self._stop.set()
-        if self._thread:
-            self._thread.join(timeout=1)
-