vermin-sdk 0.1.0__py3-none-any.whl

vermin/__init__.py

@@ -0,0 +1,90 @@
+"""Official Python SDK for Vermin (https://vermin.dev): web data for AI."""
+
+from ._base import DEFAULT_BASE_URL, CrawlOptionsDict, MapOptionsDict, ScrapeOptionsDict, SearchOptionsDict
+from ._client import AsyncCrawl, AsyncVermin, Crawl, Vermin
+from ._errors import API_ERROR_CODES, CLIENT_ERROR_CODES, VerminError
+from ._models import (
+    Action,
+    Brand,
+    BrandColor,
+    BrandFont,
+    BrandImage,
+    BrandResponse,
+    BrandSocial,
+    Chunk,
+    CrawlError,
+    CrawlEvent,
+    CrawlJob,
+    CrawlStatus,
+    Document,
+    ExtractResponse,
+    ExtractSource,
+    Location,
+    MapLink,
+    MapResponse,
+    Metadata,
+    Quality,
+    ScrapeOptions,
+    ScrapeResponse,
+    SearchHit,
+    SearchResponse,
+    VerminModel,
+    Webhook,
+)
+from ._version import __version__
+from ._webhooks import (
+    WEBHOOK_SIGNATURE_HEADER,
+    WEBHOOK_TOLERANCE_SECONDS,
+    is_valid_webhook,
+    sign_webhook,
+    verify_webhook,
+    verify_webhook_signature,
+)
+
+__all__ = [
+    "API_ERROR_CODES",
+    "CLIENT_ERROR_CODES",
+    "DEFAULT_BASE_URL",
+    "Action",
+    "AsyncCrawl",
+    "AsyncVermin",
+    "Brand",
+    "BrandColor",
+    "BrandFont",
+    "BrandImage",
+    "BrandResponse",
+    "BrandSocial",
+    "Chunk",
+    "Crawl",
+    "CrawlError",
+    "CrawlEvent",
+    "CrawlJob",
+    "CrawlOptionsDict",
+    "CrawlStatus",
+    "Document",
+    "ExtractResponse",
+    "ExtractSource",
+    "Location",
+    "MapLink",
+    "MapOptionsDict",
+    "MapResponse",
+    "Metadata",
+    "Quality",
+    "ScrapeOptions",
+    "ScrapeOptionsDict",
+    "ScrapeResponse",
+    "SearchHit",
+    "SearchOptionsDict",
+    "SearchResponse",
+    "Vermin",
+    "VerminError",
+    "VerminModel",
+    "WEBHOOK_SIGNATURE_HEADER",
+    "WEBHOOK_TOLERANCE_SECONDS",
+    "Webhook",
+    "__version__",
+    "is_valid_webhook",
+    "sign_webhook",
+    "verify_webhook",
+    "verify_webhook_signature",
+]

vermin/_base.py

@@ -0,0 +1,269 @@
+"""Shared plumbing for the sync and async clients: config, request building, retries, parsing."""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+import uuid
+from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Type, TypeVar, Union
+
+import httpx
+from pydantic import BaseModel, ValidationError
+
+from ._errors import VerminError
+from ._models import CrawlEvent, CrawlStatus, Document, ScrapeOptions, Webhook
+from ._sse import SSEMessage
+from ._version import __version__
+
+from typing_extensions import TypedDict
+
+DEFAULT_BASE_URL = "https://api.vermin.dev"
+DEFAULT_TIMEOUT = 60.0
+DEFAULT_MAX_RETRIES = 2
+
+M = TypeVar("M", bound=BaseModel)
+
+
+# ---- Keyword-argument shapes (snake_case; camelCase keys are accepted too) ---
+
+
+class ScrapeOptionsDict(TypedDict, total=False):
+    formats: List[str]
+    only_main_content: bool
+    site_markdown: bool
+    include_tags: List[str]
+    exclude_tags: List[str]
+    tier: str
+    allow_premium: bool
+    actions: List[Dict[str, Any]]
+    wait_for: int
+    timeout: int
+    """Server-side page timeout in ms (1000–60000), not the HTTP client timeout."""
+    max_age: int
+    headers: Dict[str, str]
+    location: Dict[str, Any]
+    chunk_size: int
+    ignore_robots: bool
+
+
+ScrapeOptionsInput = Union[ScrapeOptions, ScrapeOptionsDict, Mapping[str, Any]]
+
+
+class CrawlOptionsDict(TypedDict, total=False):
+    limit: int
+    max_depth: int
+    include_paths: List[str]
+    exclude_paths: List[str]
+    allow_subdomains: bool
+    sitemap: str
+    remove_boilerplate: bool
+    ignore_robots: bool
+    scrape_options: ScrapeOptionsInput
+    webhook: Union[Webhook, Mapping[str, Any]]
+
+
+class MapOptionsDict(TypedDict, total=False):
+    search: str
+    limit: int
+    include_subdomains: bool
+    sitemap: str
+    include_paths: List[str]
+    """Only return URLs whose path matches one of these globs."""
+    exclude_paths: List[str]
+    """Never return URLs whose path matches one of these globs."""
+
+
+class SearchOptionsDict(TypedDict, total=False):
+    limit: int
+    country: str
+    language: str
+    time_range: str
+    scrape_options: ScrapeOptionsInput
+
+
+# ---- Helpers -----------------------------------------------------------------
+
+
+def camel(key: str) -> str:
+    if "_" not in key:
+        return key
+    head, *rest = key.split("_")
+    return head + "".join(p[:1].upper() + p[1:] for p in rest)
+
+
+def scrape_options_wire(value: Optional[ScrapeOptionsInput]) -> Optional[Dict[str, Any]]:
+    if value is None:
+        return None
+    model = value if isinstance(value, ScrapeOptions) else ScrapeOptions.model_validate(dict(value))
+    return model.to_dict()
+
+
+def wire(options: Mapping[str, Any]) -> Dict[str, Any]:
+    """Convert snake_case keyword options to the camelCase wire body (``None`` values dropped)."""
+    body: Dict[str, Any] = {}
+    for key, value in options.items():
+        if value is None:
+            continue
+        if key in ("scrape_options", "scrapeOptions"):
+            value = scrape_options_wire(value)
+        elif key == "webhook":
+            value = (value if isinstance(value, Webhook) else Webhook.model_validate(dict(value))).to_dict()
+        body[camel(key)] = value
+    return body
+
+
+def scrape_body(url: str, options: Mapping[str, Any]) -> Dict[str, Any]:
+    return {"url": url, **ScrapeOptions.model_validate(dict(options)).to_dict()}
+
+
+def extract_schema(schema: Any) -> Optional[Dict[str, Any]]:
+    if schema is None:
+        return None
+    if isinstance(schema, type) and issubclass(schema, BaseModel):
+        return schema.model_json_schema()
+    if isinstance(schema, Mapping):
+        return dict(schema)
+    raise TypeError("schema must be a JSON Schema dict or a pydantic BaseModel subclass")
+
+
+def read_env(name: str) -> Optional[str]:
+    return os.environ.get(name) or None
+
+
+def backoff_seconds(attempt: int) -> float:
+    """Exponential backoff with jitter: 0.5s, 1s, 2s… capped at 8s, randomised to 50–100%."""
+    base: float = min(8.0, 0.5 * 2.0**attempt)
+    return base / 2 + random.random() * (base / 2)
+
+
+class Request(NamedTuple):
+    method: str
+    path: str
+    params: Optional[Dict[str, Any]] = None
+    json: Optional[Dict[str, Any]] = None
+    idempotency_key: Optional[str] = None
+    """Set for POSTs that accept an ``Idempotency-Key``; reused across retries."""
+    headers: Optional[Dict[str, str]] = None
+
+
+def idempotent(key: Optional[str]) -> str:
+    return key or str(uuid.uuid4())
+
+
+class BaseClient:
+    api_key: str
+    base_url: str
+    timeout: float
+    max_retries: int
+
+    def __init__(
+        self,
+        api_key: Optional[str],
+        base_url: Optional[str],
+        timeout: Optional[float],
+        max_retries: Optional[int],
+        headers: Optional[Mapping[str, str]],
+    ) -> None:
+        key = api_key or read_env("VERMIN_API_KEY")
+        if not key:
+            raise VerminError(
+                "missing_api_key",
+                "Missing API key: pass api_key=... or set the VERMIN_API_KEY environment variable.",
+            )
+        self.api_key = key
+        self.base_url = (base_url or read_env("VERMIN_API_URL") or DEFAULT_BASE_URL).rstrip("/")
+        self.timeout = DEFAULT_TIMEOUT if timeout is None else timeout
+        self.max_retries = max(0, DEFAULT_MAX_RETRIES if max_retries is None else max_retries)
+        self._extra_headers = dict(headers or {})
+
+    def _url(self, path: str) -> str:
+        return self.base_url + path
+
+    def _headers(self, req: Request, accept: str = "application/json") -> Dict[str, str]:
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Accept": accept,
+            "User-Agent": f"vermin-python/{__version__}",
+            **self._extra_headers,
+            **(req.headers or {}),
+        }
+        if req.idempotency_key:
+            headers["Idempotency-Key"] = req.idempotency_key
+        return headers
+
+    @staticmethod
+    def _params(req: Request) -> Optional[Dict[str, Any]]:
+        if not req.params:
+            return None
+        return {k: (str(v).lower() if isinstance(v, bool) else v) for k, v in req.params.items() if v is not None}
+
+    @staticmethod
+    def _transport_error(exc: Exception, timeout: float) -> VerminError:
+        if isinstance(exc, httpx.TimeoutException):
+            err = VerminError("timeout", f"Request timed out after {timeout}s")
+        else:
+            err = VerminError("connection_error", f"Connection error: {exc}")
+        err.__cause__ = exc
+        return err
+
+    @staticmethod
+    def _retry_delay(err: VerminError, attempt: int) -> float:
+        if err.retry_after_ms is not None:
+            return err.retry_after_ms / 1000
+        return backoff_seconds(attempt)
+
+    @staticmethod
+    def _parse(model: Type[M], response: httpx.Response) -> M:
+        try:
+            body = response.json()
+            result = model.model_validate(body)
+        except (ValueError, ValidationError) as exc:
+            err = VerminError(
+                "invalid_response",
+                f"Could not parse {model.__name__} from the response: {exc}",
+                status=response.status_code,
+                request_id=response.headers.get("x-request-id"),
+                body=response.text,
+            )
+            raise err from exc
+        if getattr(result, "request_id", "") is None and response.headers.get("x-request-id"):
+            result.request_id = response.headers["x-request-id"]  # type: ignore[attr-defined]
+        return result
+
+    @staticmethod
+    def _crawl_event(msg: SSEMessage) -> Optional[CrawlEvent]:
+        if msg.event not in ("page", "status", "done"):
+            return None
+        try:
+            data = json.loads(msg.data)
+            parsed: Union[Document, CrawlStatus] = (
+                Document.model_validate(data) if msg.event == "page" else CrawlStatus.model_validate(data)
+            )
+        except (ValueError, ValidationError) as exc:
+            raise VerminError("invalid_response", f"Invalid {msg.event!r} event: {exc}", body=msg.data) from exc
+        return CrawlEvent(type=msg.event, id=msg.id, data=parsed)
+
+    @staticmethod
+    def _validate_extract(schema: Any, response: Any) -> Any:
+        if isinstance(schema, type) and issubclass(schema, BaseModel):
+            try:
+                response.data = schema.model_validate(response.data)
+            except ValidationError as exc:
+                raise VerminError(
+                    "invalid_response",
+                    f"Extracted data did not match {schema.__name__}: {exc}",
+                    status=200,
+                    request_id=response.request_id,
+                    body=response.data,
+                ) from exc
+        return response
+
+    @staticmethod
+    def _stream_closed(reconnects: int, cause: Optional[BaseException]) -> VerminError:
+        err = VerminError(
+            "connection_error",
+            f'Crawl event stream closed before "done" (after {reconnects} reconnects)',
+        )
+        err.__cause__ = cause
+        return err