openadapt-ml 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

openadapt_ml/models/api_adapter.py

@@ -50,7 +50,9 @@ class ApiVLMAdapter(BaseVLMAdapter):
                     "Install with `uv sync --extra api`."
                 ) from exc
 
-            key = api_key or settings.anthropic_api_key or os.getenv("ANTHROPIC_API_KEY")
+            key = (
+                api_key or settings.anthropic_api_key or os.getenv("ANTHROPIC_API_KEY")
+            )
             if not key:
                 raise RuntimeError(
                     "ANTHROPIC_API_KEY is required but not found. "
@@ -87,10 +89,14 @@ class ApiVLMAdapter(BaseVLMAdapter):
         super().__init__(model=model, processor=processor, device=device)
 
     def prepare_inputs(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:  # type: ignore[override]
-        raise NotImplementedError("ApiVLMAdapter does not support training (prepare_inputs)")
+        raise NotImplementedError(
+            "ApiVLMAdapter does not support training (prepare_inputs)"
+        )
 
     def compute_loss(self, inputs: Dict[str, Any]) -> torch.Tensor:  # type: ignore[override]
-        raise NotImplementedError("ApiVLMAdapter does not support training (compute_loss)")
+        raise NotImplementedError(
+            "ApiVLMAdapter does not support training (compute_loss)"
+        )
 
     def generate(self, sample: Dict[str, Any], max_new_tokens: int = 64) -> str:  # type: ignore[override]
         images = sample.get("images", [])
@@ -138,7 +144,11 @@ class ApiVLMAdapter(BaseVLMAdapter):
 
             # Anthropic messages API returns a list of content blocks.
             parts = getattr(resp, "content", [])
-            texts = [getattr(p, "text", "") for p in parts if getattr(p, "type", "") == "text"]
+            texts = [
+                getattr(p, "text", "")
+                for p in parts
+                if getattr(p, "type", "") == "text"
+            ]
             return "\n".join([t for t in texts if t]).strip()
 
         if self.provider == "openai":

openadapt_ml/models/base_adapter.py

@@ -14,7 +14,10 @@ def get_default_device() -> torch.device:
 
     if torch.cuda.is_available():
         return torch.device("cuda")
-    if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():  # type: ignore[attr-defined]
+    if (
+        getattr(torch.backends, "mps", None) is not None
+        and torch.backends.mps.is_available()
+    ):  # type: ignore[attr-defined]
         return torch.device("mps")
     return torch.device("cpu")
 
@@ -28,7 +31,12 @@ class BaseVLMAdapter(ABC):
     - generating assistant text given a single sample at inference time
     """
 
-    def __init__(self, model: torch.nn.Module, processor: Any, device: Optional[torch.device] = None) -> None:
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        processor: Any,
+        device: Optional[torch.device] = None,
+    ) -> None:
         self.model = model
         self.processor = processor
         self.device = device or get_default_device()

openadapt_ml/models/providers/__init__.py

@@ -0,0 +1,288 @@
+"""API Provider implementations for VLM backends.
+
+This module provides a unified interface for different API providers:
+- Anthropic (Claude)
+- OpenAI (GPT)
+- Google (Gemini)
+
+The provider abstraction allows switching between different VLM backends
+without changing the calling code. Each provider handles:
+- Client creation with API key management
+- Message sending with vision support
+- Image encoding in provider-specific formats
+
+Usage:
+    from openadapt_ml.models.providers import get_provider
+
+    # Get a provider and send a message
+    provider = get_provider("anthropic")
+    client = provider.create_client(api_key)
+    response = provider.send_message(
+        client,
+        model="claude-opus-4-5-20251101",
+        system="You are a GUI agent.",
+        content=provider.build_content(
+            text="Click the submit button",
+            image=screenshot,
+        ),
+    )
+
+    # Or use the quick_message helper
+    response = provider.quick_message(
+        api_key=key,
+        model="claude-opus-4-5-20251101",
+        prompt="What's in this image?",
+        image=screenshot,
+    )
+
+Model Aliases:
+    Common model aliases are provided for convenience:
+    - "claude-opus-4.5" -> ("anthropic", "claude-opus-4-5-20251101")
+    - "gpt-5.2" -> ("openai", "gpt-5.2")
+    - "gemini-3-pro" -> ("google", "gemini-3-pro")
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from openadapt_ml.models.providers.base import (
+    BaseAPIProvider,
+    ProviderError,
+    AuthenticationError,
+    RateLimitError,
+    ModelNotFoundError,
+)
+from openadapt_ml.models.providers.anthropic import AnthropicProvider
+from openadapt_ml.models.providers.openai import OpenAIProvider
+from openadapt_ml.models.providers.google import GoogleProvider
+
+if TYPE_CHECKING:
+    from PIL import Image
+
+__all__ = [
+    # Base classes and exceptions
+    "BaseAPIProvider",
+    "ProviderError",
+    "AuthenticationError",
+    "RateLimitError",
+    "ModelNotFoundError",
+    # Provider implementations
+    "AnthropicProvider",
+    "OpenAIProvider",
+    "GoogleProvider",
+    # Factory functions
+    "get_provider",
+    "get_provider_for_model",
+    "resolve_model_alias",
+    # Registries
+    "PROVIDERS",
+    "MODEL_ALIASES",
+    # Convenience functions
+    "quick_message",
+    "list_providers",
+    "list_models",
+]
+
+# Provider registry
+PROVIDERS: dict[str, type[BaseAPIProvider]] = {
+    "anthropic": AnthropicProvider,
+    "openai": OpenAIProvider,
+    "google": GoogleProvider,
+}
+
+# Model aliases for convenience
+# Maps friendly names to (provider, model_id) tuples
+MODEL_ALIASES: dict[str, tuple[str, str]] = {
+    # Anthropic
+    "claude-opus-4.5": ("anthropic", "claude-opus-4-5-20251101"),
+    "claude-sonnet-4.5": ("anthropic", "claude-sonnet-4-5-20250929"),
+    "claude-haiku-3.5": ("anthropic", "claude-haiku-3-5-20241022"),
+    # OpenAI
+    "gpt-5.2": ("openai", "gpt-5.2"),
+    "gpt-5.1": ("openai", "gpt-5.1"),
+    "gpt-4o": ("openai", "gpt-4o"),
+    "gpt-4o-mini": ("openai", "gpt-4o-mini"),
+    # Google
+    "gemini-3-pro": ("google", "gemini-3-pro"),
+    "gemini-3-flash": ("google", "gemini-3-flash"),
+    "gemini-2.5-pro": ("google", "gemini-2.5-pro"),
+    "gemini-2.5-flash": ("google", "gemini-2.5-flash"),
+}
+
+
+def get_provider(provider_name: str) -> BaseAPIProvider:
+    """Get a provider instance by name.
+
+    Args:
+        provider_name: Provider identifier ('anthropic', 'openai', 'google').
+
+    Returns:
+        Provider instance.
+
+    Raises:
+        ValueError: If provider_name is not recognized.
+
+    Example:
+        >>> provider = get_provider("anthropic")
+        >>> provider.name
+        'anthropic'
+    """
+    provider_class = PROVIDERS.get(provider_name.lower())
+    if provider_class is None:
+        available = ", ".join(PROVIDERS.keys())
+        raise ValueError(f"Unknown provider: {provider_name}. Available: {available}")
+    return provider_class()
+
+
+def resolve_model_alias(alias: str) -> tuple[str, str]:
+    """Resolve a model alias to (provider, model_id).
+
+    Args:
+        alias: Model alias (e.g., 'claude-opus-4.5') or full model ID.
+
+    Returns:
+        Tuple of (provider_name, model_id).
+
+    Raises:
+        ValueError: If alias is not recognized and can't be inferred.
+
+    Example:
+        >>> resolve_model_alias("claude-opus-4.5")
+        ('anthropic', 'claude-opus-4-5-20251101')
+        >>> resolve_model_alias("gemini-3-pro")
+        ('google', 'gemini-3-pro')
+    """
+    # Check explicit aliases first
+    if alias in MODEL_ALIASES:
+        return MODEL_ALIASES[alias]
+
+    # Try to infer provider from model name patterns
+    alias_lower = alias.lower()
+
+    if alias_lower.startswith("claude"):
+        return ("anthropic", alias)
+    elif alias_lower.startswith("gpt"):
+        return ("openai", alias)
+    elif alias_lower.startswith("gemini"):
+        return ("google", alias)
+
+    raise ValueError(
+        f"Unknown model alias: {alias}. "
+        f"Available aliases: {', '.join(MODEL_ALIASES.keys())}. "
+        f"Or use a full model ID with a known prefix (claude-*, gpt-*, gemini-*)."
+    )
+
+
+def get_provider_for_model(model: str) -> tuple[BaseAPIProvider, str]:
+    """Get the appropriate provider for a model.
+
+    Args:
+        model: Model alias or full model ID.
+
+    Returns:
+        Tuple of (provider_instance, resolved_model_id).
+
+    Example:
+        >>> provider, model_id = get_provider_for_model("claude-opus-4.5")
+        >>> provider.name
+        'anthropic'
+        >>> model_id
+        'claude-opus-4-5-20251101'
+    """
+    provider_name, model_id = resolve_model_alias(model)
+    provider = get_provider(provider_name)
+    return provider, model_id
+
+
+def quick_message(
+    model: str,
+    prompt: str,
+    image: "Image | None" = None,
+    system: str = "",
+    api_key: str | None = None,
+    max_tokens: int = 1024,
+    temperature: float = 0.1,
+) -> str:
+    """Send a quick message to any model.
+
+    Convenience function that resolves the provider, creates a client,
+    and sends a message in one call. Useful for one-off requests.
+
+    Args:
+        model: Model alias or full model ID.
+        prompt: User prompt text.
+        image: Optional image to include.
+        system: Optional system prompt.
+        api_key: Optional API key (uses settings/env if not provided).
+        max_tokens: Maximum tokens in response.
+        temperature: Sampling temperature.
+
+    Returns:
+        Model response text.
+
+    Raises:
+        AuthenticationError: If no API key is available.
+        ProviderError: For API errors.
+
+    Example:
+        >>> response = quick_message(
+        ...     model="claude-opus-4.5",
+        ...     prompt="What's in this image?",
+        ...     image=screenshot,
+        ... )
+    """
+    provider, model_id = get_provider_for_model(model)
+    resolved_key = provider.get_api_key(api_key)
+    return provider.quick_message(
+        api_key=resolved_key,
+        model=model_id,
+        prompt=prompt,
+        image=image,
+        system=system,
+        max_tokens=max_tokens,
+        temperature=temperature,
+    )
+
+
+def list_providers() -> list[str]:
+    """List available provider names.
+
+    Returns:
+        List of provider identifiers.
+
+    Example:
+        >>> list_providers()
+        ['anthropic', 'openai', 'google']
+    """
+    return list(PROVIDERS.keys())
+
+
+def list_models(provider: str | None = None) -> dict[str, dict]:
+    """List available models, optionally filtered by provider.
+
+    Args:
+        provider: Optional provider name to filter by.
+
+    Returns:
+        Dict mapping model IDs to their properties.
+
+    Example:
+        >>> list_models("anthropic")
+        {
+            'claude-opus-4-5-20251101': {'context': 200000, 'description': 'SOTA computer use'},
+            ...
+        }
+    """
+    if provider:
+        provider_instance = get_provider(provider)
+        return provider_instance.supported_models
+
+    # Combine models from all providers
+    all_models = {}
+    for provider_name in PROVIDERS:
+        provider_instance = get_provider(provider_name)
+        for model_id, props in provider_instance.supported_models.items():
+            all_models[model_id] = {**props, "provider": provider_name}
+
+    return all_models

openadapt_ml/models/providers/anthropic.py

@@ -0,0 +1,266 @@
+"""Anthropic (Claude) API provider.
+
+Supports Claude Opus 4.5, Sonnet 4.5, and other Claude models.
+Implements the BaseAPIProvider interface for the Anthropic Messages API.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from openadapt_ml.models.providers.base import (
+    BaseAPIProvider,
+    AuthenticationError,
+    ModelNotFoundError,
+    ProviderError,
+    RateLimitError,
+)
+
+if TYPE_CHECKING:
+    from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+# Default models
+DEFAULT_MODEL = "claude-sonnet-4-5-20250929"
+
+# Supported models with their context windows
+SUPPORTED_MODELS = {
+    "claude-opus-4-5-20251101": {
+        "context": 200_000,
+        "description": "SOTA computer use",
+    },
+    "claude-sonnet-4-5-20250929": {"context": 200_000, "description": "Fast, cheaper"},
+    "claude-sonnet-4-20250514": {"context": 200_000, "description": "Previous Sonnet"},
+    "claude-haiku-3-5-20241022": {
+        "context": 200_000,
+        "description": "Fastest, cheapest",
+    },
+}
+
+
+class AnthropicProvider(BaseAPIProvider):
+    """Provider for Anthropic's Claude models.
+
+    Implements vision support via base64-encoded images in the Messages API format.
+    Claude models natively support screenshots and UI analysis for computer use tasks.
+
+    Supported models:
+        - claude-opus-4-5-20251101: Most capable, best for complex GUI tasks
+        - claude-sonnet-4-5-20250929: Fast and cost-effective
+        - claude-haiku-3-5-20241022: Fastest, lowest cost
+
+    Example:
+        >>> provider = AnthropicProvider()
+        >>> client = provider.create_client(api_key)
+        >>> response = provider.send_message(
+        ...     client,
+        ...     model="claude-opus-4-5-20251101",
+        ...     system="You are a GUI agent.",
+        ...     content=[
+        ...         {"type": "text", "text": "Click the submit button"},
+        ...         provider.encode_image(screenshot),
+        ...     ],
+        ... )
+
+    Attributes:
+        name: Returns 'anthropic'.
+    """
+
+    @property
+    def name(self) -> str:
+        """Provider name."""
+        return "anthropic"
+
+    @property
+    def env_key_name(self) -> str:
+        """Environment variable name for API key."""
+        return "ANTHROPIC_API_KEY"
+
+    @property
+    def default_model(self) -> str:
+        """Default model to use."""
+        return DEFAULT_MODEL
+
+    @property
+    def supported_models(self) -> dict[str, dict[str, Any]]:
+        """Dictionary of supported models and their properties."""
+        return SUPPORTED_MODELS
+
+    def create_client(self, api_key: str) -> Any:
+        """Create Anthropic client.
+
+        Args:
+            api_key: Anthropic API key.
+
+        Returns:
+            Anthropic client instance.
+
+        Raises:
+            ImportError: If anthropic package not installed.
+            AuthenticationError: If API key format is invalid.
+        """
+        try:
+            from anthropic import Anthropic
+        except ImportError as e:
+            raise ImportError(
+                "anthropic package is required for provider='anthropic'. "
+                "Install with: uv add anthropic"
+            ) from e
+
+        if not api_key or not api_key.strip():
+            raise AuthenticationError(
+                "Anthropic API key cannot be empty. "
+                "Get a key from https://console.anthropic.com/"
+            )
+
+        logger.debug("Creating Anthropic client")
+        return Anthropic(api_key=api_key)
+
+    def send_message(
+        self,
+        client: Any,
+        model: str,
+        system: str,
+        content: list[dict[str, Any]],
+        max_tokens: int = 1024,
+        temperature: float = 0.1,
+    ) -> str:
+        """Send message using Anthropic Messages API.
+
+        Args:
+            client: Anthropic client from create_client().
+            model: Model ID (e.g., 'claude-opus-4-5-20251101').
+            system: System prompt.
+            content: List of content blocks (text and images).
+            max_tokens: Max response tokens.
+            temperature: Sampling temperature (0.0-1.0).
+
+        Returns:
+            Model response text.
+
+        Raises:
+            AuthenticationError: If API key is invalid.
+            RateLimitError: If rate limit exceeded.
+            ModelNotFoundError: If model doesn't exist.
+            ProviderError: For other API errors.
+        """
+        logger.debug(f"Sending message to {model} with {len(content)} content blocks")
+
+        try:
+            response = client.messages.create(
+                model=model,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                system=system or None,
+                messages=[{"role": "user", "content": content}],
+            )
+
+            # Extract text from content blocks
+            parts = getattr(response, "content", [])
+            texts = [
+                getattr(p, "text", "")
+                for p in parts
+                if getattr(p, "type", "") == "text"
+            ]
+            result = "\n".join([t for t in texts if t]).strip()
+
+            logger.debug(f"Received response: {len(result)} chars")
+            return result
+
+        except Exception as e:
+            error_str = str(e).lower()
+
+            # Map common errors to specific exceptions
+            if "authentication" in error_str or "api_key" in error_str:
+                raise AuthenticationError(
+                    f"Anthropic authentication failed: {e}"
+                ) from e
+            elif "rate_limit" in error_str or "429" in error_str:
+                raise RateLimitError(f"Anthropic rate limit exceeded: {e}") from e
+            elif "model_not_found" in error_str or "not found" in error_str:
+                raise ModelNotFoundError(f"Model '{model}' not found: {e}") from e
+            else:
+                raise ProviderError(f"Anthropic API error: {e}") from e
+
+    def encode_image(self, image: "Image") -> dict[str, Any]:
+        """Encode image for Anthropic API.
+
+        Anthropic uses base64-encoded images with explicit source type.
+        PNG format is used for lossless quality.
+
+        Args:
+            image: PIL Image.
+
+        Returns:
+            Image content block for Anthropic API in format:
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": "image/png",
+                    "data": "<base64_string>"
+                }
+            }
+        """
+        return {
+            "type": "image",
+            "source": {
+                "type": "base64",
+                "media_type": "image/png",
+                "data": self.image_to_base64(image, "PNG"),
+            },
+        }
+
+    def encode_image_from_bytes(
+        self,
+        image_bytes: bytes,
+        media_type: str = "image/png",
+    ) -> dict[str, Any]:
+        """Encode raw image bytes for Anthropic API.
+
+        Useful when you already have image bytes and don't need PIL.
+
+        Args:
+            image_bytes: Raw image bytes.
+            media_type: MIME type of the image.
+
+        Returns:
+            Image content block for Anthropic API.
+        """
+        import base64
+
+        return {
+            "type": "image",
+            "source": {
+                "type": "base64",
+                "media_type": media_type,
+                "data": base64.b64encode(image_bytes).decode("utf-8"),
+            },
+        }
+
+    def encode_image_from_url(self, url: str) -> dict[str, Any]:
+        """Create image content block from URL.
+
+        Note: Anthropic doesn't support URL-based images directly.
+        This method fetches the URL and encodes the image.
+
+        Args:
+            url: Image URL to fetch and encode.
+
+        Returns:
+            Image content block for Anthropic API.
+
+        Raises:
+            ProviderError: If URL fetch fails.
+        """
+        import urllib.request
+
+        try:
+            with urllib.request.urlopen(url) as response:
+                image_bytes = response.read()
+                content_type = response.headers.get("Content-Type", "image/png")
+                return self.encode_image_from_bytes(image_bytes, content_type)
+        except Exception as e:
+            raise ProviderError(f"Failed to fetch image from URL: {e}") from e