openadapt-ml 0.1.0__py3-none-any.whl

openadapt_ml/models/api_adapter.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import base64
+import os
+
+import torch
+
+from openadapt_ml.config import settings
+from openadapt_ml.models.base_adapter import BaseVLMAdapter, get_default_device
+
+
+class ApiVLMAdapter(BaseVLMAdapter):
+    """Inference-only adapter for hosted VLM APIs (Anthropic, OpenAI).
+
+    This adapter implements `generate` only; `prepare_inputs` and
+    `compute_loss` are not supported and will raise NotImplementedError.
+    """
+
+    def __init__(
+        self,
+        provider: str,
+        device: Optional[torch.device] = None,
+        api_key: Optional[str] = None,
+    ) -> None:
+        """Initialize an API-backed adapter.
+
+        Parameters
+        ----------
+        provider:
+            "anthropic" or "openai".
+        device:
+            Unused for remote APIs but kept for BaseVLMAdapter compatibility.
+        api_key:
+            Optional API key override. If not provided, keys are loaded from:
+            1. Settings (.env file)
+            2. Environment variables (ANTHROPIC_API_KEY / OPENAI_API_KEY)
+            3. Error if not found
+        """
+
+        self.provider = provider
+
+        if provider == "anthropic":
+            try:
+                from anthropic import Anthropic  # type: ignore[import]
+            except Exception as exc:  # pragma: no cover - import-time failure
+                raise RuntimeError(
+                    "anthropic package is required for provider='anthropic'. "
+                    "Install with `uv sync --extra api`."
+                ) from exc
+
+            key = api_key or settings.anthropic_api_key or os.getenv("ANTHROPIC_API_KEY")
+            if not key:
+                raise RuntimeError(
+                    "ANTHROPIC_API_KEY is required but not found. "
+                    "Please set it in .env file, environment variable, or pass api_key parameter."
+                )
+            client = Anthropic(api_key=key)
+        elif provider == "openai":
+            try:
+                from openai import OpenAI  # type: ignore[import]
+            except Exception as exc:  # pragma: no cover - import-time failure
+                raise RuntimeError(
+                    "openai package is required for provider='openai'. "
+                    "Install with `uv sync --extra api`."
+                ) from exc
+
+            key = api_key or settings.openai_api_key or os.getenv("OPENAI_API_KEY")
+            if not key:
+                raise RuntimeError(
+                    "OPENAI_API_KEY is required but not found. "
+                    "Please set it in .env file, environment variable, or pass api_key parameter."
+                )
+            client = OpenAI(api_key=key)
+        else:
+            raise ValueError(f"Unsupported provider: {provider}")
+
+        if device is None:
+            device = get_default_device()
+
+        # Store client separately; BaseVLMAdapter expects a model + processor, so
+        # we pass a tiny dummy module and the client as the "processor".
+        self._client = client
+        model = torch.nn.Identity()
+        processor: Any = client
+        super().__init__(model=model, processor=processor, device=device)
+
+    def prepare_inputs(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:  # type: ignore[override]
+        raise NotImplementedError("ApiVLMAdapter does not support training (prepare_inputs)")
+
+    def compute_loss(self, inputs: Dict[str, Any]) -> torch.Tensor:  # type: ignore[override]
+        raise NotImplementedError("ApiVLMAdapter does not support training (compute_loss)")
+
+    def generate(self, sample: Dict[str, Any], max_new_tokens: int = 64) -> str:  # type: ignore[override]
+        images = sample.get("images", [])
+        if not images:
+            raise ValueError("Sample is missing image paths")
+        image_path = images[0]
+
+        messages = sample.get("messages", [])
+        system_text = ""
+        user_text = ""
+        for m in messages:
+            role = m.get("role")
+            if role == "system":
+                system_text = m.get("content", "")
+            elif role == "user":
+                user_text = m.get("content", "")
+
+        with open(image_path, "rb") as f:
+            image_bytes = f.read()
+
+        if self.provider == "anthropic":
+            client: Any = self._client
+            image_b64 = base64.b64encode(image_bytes).decode("utf-8")
+
+            content: List[Dict[str, Any]] = []
+            if user_text:
+                content.append({"type": "text", "text": user_text})
+            content.append(
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": image_b64,
+                    },
+                }
+            )
+
+            resp = client.messages.create(
+                model="claude-sonnet-4-5-20250929",
+                max_tokens=max_new_tokens,
+                system=system_text or None,
+                messages=[{"role": "user", "content": content}],
+            )
+
+            # Anthropic messages API returns a list of content blocks.
+            parts = getattr(resp, "content", [])
+            texts = [getattr(p, "text", "") for p in parts if getattr(p, "type", "") == "text"]
+            return "\n".join([t for t in texts if t]).strip()
+
+        if self.provider == "openai":
+            client: Any = self._client
+            base64_image = base64.b64encode(image_bytes).decode("utf-8")
+
+            messages_payload: List[Dict[str, Any]] = []
+            if system_text:
+                messages_payload.append({"role": "system", "content": system_text})
+
+            user_content: List[Dict[str, Any]] = []
+            if user_text:
+                user_content.append({"type": "text", "text": user_text})
+            user_content.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{base64_image}"},
+                }
+            )
+            messages_payload.append({"role": "user", "content": user_content})
+
+            resp = client.chat.completions.create(
+                model="gpt-5.1",
+                messages=messages_payload,
+                max_completion_tokens=max_new_tokens,
+            )
+            return resp.choices[0].message.content or ""
+
+        # Should be unreachable because provider is validated in __init__.
+        raise ValueError(f"Unsupported provider: {self.provider}")

openadapt_ml/models/base_adapter.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+import torch
+
+
+def get_default_device() -> torch.device:
+    """Select cuda, then mps, then cpu.
+
+    This is used as a fallback when no explicit device is provided.
+    """
+
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():  # type: ignore[attr-defined]
+        return torch.device("mps")
+    return torch.device("cpu")
+
+
+class BaseVLMAdapter(ABC):
+    """Abstract wrapper around a vision-language model + processor.
+
+    Concrete implementations are responsible for:
+    - converting SFT-style samples into model inputs (tokenization, image processing)
+    - computing supervised training loss
+    - generating assistant text given a single sample at inference time
+    """
+
+    def __init__(self, model: torch.nn.Module, processor: Any, device: Optional[torch.device] = None) -> None:
+        self.model = model
+        self.processor = processor
+        self.device = device or get_default_device()
+        self.model.to(self.device)
+
+    @abstractmethod
+    def prepare_inputs(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Convert a batch of SFT samples into model inputs.
+
+        The batch is a list of samples of the form produced by
+        `build_next_action_sft_samples` (images + messages).
+        Implementations should return a dict suitable for passing to the
+        underlying HF model, including `labels` for supervised loss.
+        """
+
+    @abstractmethod
+    def compute_loss(self, inputs: Dict[str, Any]) -> torch.Tensor:
+        """Run the model forward and return a scalar loss tensor."""
+
+    @abstractmethod
+    def generate(self, sample: Dict[str, Any], max_new_tokens: int = 64) -> str:
+        """Generate assistant text for a single SFT-style sample."""
+
+    def train(self) -> None:
+        self.model.train()
+
+    def eval(self) -> None:
+        self.model.eval()

openadapt_ml/models/dummy_adapter.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import torch
+from torch import nn
+
+from openadapt_ml.models.base_adapter import BaseVLMAdapter, get_default_device
+
+
+class DummyAdapter(BaseVLMAdapter):
+    """Minimal adapter used to validate the training loop.
+
+    - Ignores images/messages content.
+    - Uses a tiny linear model and returns a simple MSE loss.
+    - generate() returns a fixed string.
+    """
+
+    def __init__(self, device: Optional[torch.device] = None) -> None:
+        if device is None:
+            device = get_default_device()
+        # Tiny dummy model with a few parameters
+        model = nn.Linear(4, 1)
+        processor: Any = None
+        super().__init__(model=model, processor=processor, device=device)
+
+    def prepare_inputs(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:  # type: ignore[override]
+        batch_size = len(batch)
+        # Create a dummy input tensor; real adapters will encode images + text.
+        x = torch.zeros(batch_size, 4, device=self.device)
+        # Target is a constant zero tensor; loss will be ||model(x)||^2.
+        y = torch.zeros(batch_size, 1, device=self.device)
+        return {"inputs": x, "targets": y}
+
+    def compute_loss(self, inputs: Dict[str, Any]) -> torch.Tensor:  # type: ignore[override]
+        x = inputs["inputs"]
+        y = inputs["targets"]
+        preds = self.model(x)
+        return torch.mean((preds - y) ** 2)
+
+    def generate(self, sample: Dict[str, Any], max_new_tokens: int = 64) -> str:  # type: ignore[override]
+        return "DONE()"