bead 0.1.0__py3-none-any.whl

bead/items/adapters/anthropic.py

@@ -0,0 +1,224 @@
+"""Anthropic API adapter for item construction.
+
+This module provides a ModelAdapter implementation for Anthropic's Claude API,
+supporting natural language inference via prompting. Note that Claude API does
+not provide direct access to log probabilities or embeddings.
+"""
+
+from __future__ import annotations
+
+import os
+
+import numpy as np
+
+try:
+    import anthropic
+except ImportError as e:
+    raise ImportError(
+        "anthropic package is required for Anthropic adapter. "
+        "Install it with: pip install anthropic"
+    ) from e
+
+from bead.items.adapters.api_utils import rate_limit, retry_with_backoff
+from bead.items.adapters.base import ModelAdapter
+from bead.items.cache import ModelOutputCache
+
+
+class AnthropicAdapter(ModelAdapter):
+    """Adapter for Anthropic Claude API models.
+
+    Provides access to Claude models for prompted natural language inference.
+    Note that Claude API does not support log probability computation or
+    embeddings, so those methods will raise NotImplementedError.
+
+    Parameters
+    ----------
+    model_name : str
+        Claude model identifier (default: "claude-3-5-sonnet-20241022").
+    api_key : str | None
+        Anthropic API key. If None, uses ANTHROPIC_API_KEY environment variable.
+    cache : ModelOutputCache | None
+        Cache for model outputs. If None, creates in-memory cache.
+    model_version : str
+        Model version for cache tracking (default: "latest").
+
+    Attributes
+    ----------
+    model_name : str
+        Claude model identifier (e.g., "claude-3-5-sonnet-20241022").
+    client : anthropic.Anthropic
+        Anthropic API client.
+
+    Raises
+    ------
+    ValueError
+        If no API key is provided and ANTHROPIC_API_KEY is not set.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "claude-3-5-sonnet-20241022",
+        api_key: str | None = None,
+        cache: ModelOutputCache | None = None,
+        model_version: str = "latest",
+    ) -> None:
+        if cache is None:
+            cache = ModelOutputCache(backend="memory")
+
+        super().__init__(
+            model_name=model_name, cache=cache, model_version=model_version
+        )
+
+        # Get API key from parameter or environment
+        if api_key is None:
+            api_key = os.environ.get("ANTHROPIC_API_KEY")
+            if api_key is None:
+                raise ValueError(
+                    "Anthropic API key must be provided via api_key parameter "
+                    "or ANTHROPIC_API_KEY environment variable"
+                )
+
+        self.client = anthropic.Anthropic(api_key=api_key)
+
+    def compute_log_probability(self, text: str) -> float:
+        """Compute log probability of text.
+
+        Not supported by Anthropic API.
+
+        Raises
+        ------
+        NotImplementedError
+            Always raised - Claude API does not provide log probabilities.
+        """
+        raise NotImplementedError(
+            "Log probability computation is not supported by Anthropic Claude API. "
+            "Claude does not provide access to token-level probabilities."
+        )
+
+    def compute_perplexity(self, text: str) -> float:
+        """Compute perplexity of text.
+
+        Not supported by Anthropic API (requires log probabilities).
+
+        Raises
+        ------
+        NotImplementedError
+            Always raised - requires log probability support.
+        """
+        raise NotImplementedError(
+            "Perplexity computation is not supported by Anthropic Claude API. "
+            "This operation requires log probabilities, which Claude does not provide."
+        )
+
+    def get_embedding(self, text: str) -> np.ndarray:
+        """Get embedding vector for text.
+
+        Not supported by Anthropic API.
+
+        Raises
+        ------
+        NotImplementedError
+            Always raised - Claude API does not provide embeddings.
+        """
+        raise NotImplementedError(
+            "Embedding computation is not supported by Anthropic Claude API. "
+            "Claude does not provide embedding vectors. "
+            "Consider using OpenAI's text-embedding models or sentence transformers."
+        )
+
+    @retry_with_backoff(
+        max_retries=3,
+        initial_delay=1.0,
+        backoff_factor=2.0,
+        exceptions=(
+            anthropic.APIError,
+            anthropic.APIConnectionError,
+            anthropic.RateLimitError,
+        ),
+    )
+    @rate_limit(calls_per_minute=60)
+    def compute_nli(self, premise: str, hypothesis: str) -> dict[str, float]:
+        """Compute natural language inference scores via prompting.
+
+        Uses Claude's messages API with a prompt to classify the relationship
+        between premise and hypothesis.
+
+        Parameters
+        ----------
+        premise : str
+            Premise text.
+        hypothesis : str
+            Hypothesis text.
+
+        Returns
+        -------
+        dict[str, float]
+            Dictionary with keys "entailment", "neutral", "contradiction"
+            mapping to probability scores.
+        """
+        # Check cache
+        cached = self.cache.get(
+            model_name=self.model_name,
+            operation="nli",
+            premise=premise,
+            hypothesis=hypothesis,
+        )
+        if cached is not None:
+            return dict(cached)
+
+        # Construct prompt
+        prompt = (
+            "Given the following premise and hypothesis, "
+            "determine the relationship between them.\n\n"
+            f"Premise: {premise}\n"
+            f"Hypothesis: {hypothesis}\n\n"
+            "Choose one of the following:\n"
+            "- entailment: The hypothesis is definitely true given the premise\n"
+            "- neutral: The hypothesis might be true given the premise\n"
+            "- contradiction: The hypothesis is definitely false given the premise\n\n"
+            "Respond with only one word: entailment, neutral, or contradiction."
+        )
+
+        # Call API
+        response = self.client.messages.create(
+            model=self.model_name,
+            max_tokens=10,
+            temperature=0.0,
+            messages=[{"role": "user", "content": prompt}],
+        )
+
+        # Parse response
+        if not response.content or len(response.content) == 0:
+            raise ValueError("API response did not include content")
+
+        # Get text from first content block
+        answer = response.content[0].text.strip().lower()
+
+        # Map to scores
+        scores: dict[str, float] = {
+            "entailment": 0.0,
+            "neutral": 0.0,
+            "contradiction": 0.0,
+        }
+
+        if "entailment" in answer:
+            scores["entailment"] = 1.0
+        elif "neutral" in answer:
+            scores["neutral"] = 1.0
+        elif "contradiction" in answer:
+            scores["contradiction"] = 1.0
+        else:
+            # Default to neutral if unclear
+            scores["neutral"] = 1.0
+
+        # Cache result
+        self.cache.set(
+            model_name=self.model_name,
+            operation="nli",
+            result=scores,
+            model_version=self.model_version,
+            premise=premise,
+            hypothesis=hypothesis,
+        )
+
+        return scores

bead/items/adapters/api_utils.py

@@ -0,0 +1,167 @@
+"""Utilities for API-based model adapters.
+
+This module provides shared utilities for API-based model adapters,
+including retry logic with exponential backoff and rate limiting.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from functools import wraps
+from typing import ParamSpec, TypeVar
+
+P = ParamSpec("P")
+
+T = TypeVar("T")
+
+
+def retry_with_backoff(
+    max_retries: int = 3,
+    initial_delay: float = 1.0,
+    backoff_factor: float = 2.0,
+    exceptions: tuple[type[Exception], ...] = (Exception,),
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """Decorate function with retry logic and exponential backoff.
+
+    Retries a function call on specified exceptions with exponential backoff
+    between attempts. The delay between retries grows exponentially:
+    delay = initial_delay * (backoff_factor ** attempt).
+
+    Parameters
+    ----------
+    max_retries : int
+        Maximum number of retry attempts (default: 3).
+    initial_delay : float
+        Initial delay in seconds before first retry (default: 1.0).
+    backoff_factor : float
+        Multiplicative factor for delay between retries (default: 2.0).
+    exceptions : tuple[type[Exception], ...]
+        Tuple of exception types to catch and retry on (default: (Exception,)).
+
+    Returns
+    -------
+    Callable
+        Decorated function with retry logic.
+
+    Examples
+    --------
+    >>> @retry_with_backoff(max_retries=3, initial_delay=1.0)
+    ... def call_api():
+    ...     # May raise transient errors
+    ...     return api.get_data()
+    """
+
+    def decorator(func: Callable[P, T]) -> Callable[P, T]:
+        @wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            last_exception: Exception | None = None
+
+            for attempt in range(max_retries + 1):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    last_exception = e
+                    if attempt < max_retries:
+                        delay = initial_delay * (backoff_factor**attempt)
+                        time.sleep(delay)
+                    else:
+                        # last attempt failed, re-raise
+                        raise
+
+            # should never reach here, but for type checker
+            if last_exception is not None:
+                raise last_exception
+            raise RuntimeError("Unexpected state in retry_with_backoff")
+
+        return wrapper
+
+    return decorator
+
+
+class RateLimiter:
+    """Rate limiter for API calls.
+
+    Tracks call timestamps and enforces a maximum rate of calls per minute.
+    Uses a sliding window algorithm to ensure the rate limit is respected.
+
+    Parameters
+    ----------
+    calls_per_minute : int
+        Maximum number of calls allowed per minute (default: 60).
+
+    Attributes
+    ----------
+    calls_per_minute : int
+        Maximum number of calls allowed per minute.
+    call_times : list[float]
+        Timestamps of recent API calls.
+    """
+
+    def __init__(self, calls_per_minute: int = 60) -> None:
+        self.calls_per_minute = calls_per_minute
+        self.call_times: list[float] = []
+
+    def wait_if_needed(self) -> None:
+        """Wait if rate limit would be exceeded.
+
+        Checks if making a call now would exceed the rate limit.
+        If so, sleeps until enough time has passed.
+        """
+        now = time.time()
+
+        # remove calls older than 1 minute
+        cutoff_time = now - 60.0
+        self.call_times = [t for t in self.call_times if t > cutoff_time]
+
+        # if at rate limit, wait until oldest call expires
+        if len(self.call_times) >= self.calls_per_minute:
+            oldest_call = self.call_times[0]
+            wait_time = 60.0 - (now - oldest_call)
+            if wait_time > 0:
+                time.sleep(wait_time)
+            # clean up again after waiting
+            now = time.time()
+            cutoff_time = now - 60.0
+            self.call_times = [t for t in self.call_times if t > cutoff_time]
+
+        # record this call
+        self.call_times.append(time.time())
+
+
+def rate_limit(
+    calls_per_minute: int = 60,
+) -> Callable[[Callable[P, T]], Callable[P, T]]:
+    """Decorate function with rate limiting for API calls.
+
+    Enforces a maximum rate of API calls per minute using a shared
+    RateLimiter instance. Calls that would exceed the rate limit
+    will block until the limit resets.
+
+    Parameters
+    ----------
+    calls_per_minute : int
+        Maximum number of calls allowed per minute (default: 60).
+
+    Returns
+    -------
+    Callable
+        Decorated function with rate limiting.
+
+    Examples
+    --------
+    >>> @rate_limit(calls_per_minute=30)
+    ... def call_api():
+    ...     return api.get_data()
+    """
+    limiter = RateLimiter(calls_per_minute=calls_per_minute)
+
+    def decorator(func: Callable[P, T]) -> Callable[P, T]:
+        @wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            limiter.wait_if_needed()
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

bead/items/adapters/base.py

@@ -0,0 +1,216 @@
+"""Base class for model adapters used in item construction.
+
+This module defines the abstract ModelAdapter interface that all model adapters
+must implement to support judgment prediction operations during Stage 3
+(Item Construction).
+
+This is SEPARATE from template filling model adapters
+(bead.templates.models.adapter), which are used in Stage 2.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+from bead.items.cache import ModelOutputCache
+
+
+class ModelAdapter(ABC):
+    """Base class for model adapters used in item construction.
+
+    All model adapters must implement this interface to support
+    judgment prediction operations during Stage 3 (Item Construction).
+
+    This is SEPARATE from template filling model adapters
+    (bead.templates.models.adapter), which are used in Stage 2.
+
+    Parameters
+    ----------
+    model_name : str
+        Model identifier (e.g., "gpt2", "roberta-large-mnli").
+    cache : ModelOutputCache
+        Cache instance for storing model outputs.
+    model_version : str
+        Version of the model for cache tracking.
+
+    Attributes
+    ----------
+    model_name : str
+        Model identifier (e.g., "gpt2", "roberta-large-mnli").
+    model_version : str
+        Version of the model.
+    cache : ModelOutputCache
+        Cache for model outputs.
+    """
+
+    def __init__(
+        self, model_name: str, cache: ModelOutputCache, model_version: str = "unknown"
+    ) -> None:
+        self.model_name = model_name
+        self.model_version = model_version
+        self.cache = cache
+
+    @abstractmethod
+    def compute_log_probability(self, text: str) -> float:
+        """Compute log probability of text under language model.
+
+        Required for language model constraints. Should raise NotImplementedError
+        if not supported by model type.
+
+        Parameters
+        ----------
+        text : str
+            Text to compute log probability for.
+
+        Returns
+        -------
+        float
+            Log probability of the text.
+
+        Raises
+        ------
+        NotImplementedError
+            If this operation is not supported by the model type.
+        """
+        pass
+
+    @abstractmethod
+    def compute_perplexity(self, text: str) -> float:
+        """Compute perplexity of text.
+
+        Required for complexity-based filtering. Should raise NotImplementedError
+        if not supported by model type.
+
+        Parameters
+        ----------
+        text : str
+            Text to compute perplexity for.
+
+        Returns
+        -------
+        float
+            Perplexity of the text (must be positive).
+
+        Raises
+        ------
+        NotImplementedError
+            If this operation is not supported by the model type.
+        """
+        pass
+
+    @abstractmethod
+    def get_embedding(
+        self, text: str
+    ) -> np.ndarray[tuple[int, ...], np.dtype[np.float64]]:
+        """Get embedding vector for text.
+
+        Required for similarity computations and semantic clustering.
+        Should raise NotImplementedError if not supported by model type.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+
+        Returns
+        -------
+        np.ndarray
+            Embedding vector for the text.
+
+        Raises
+        ------
+        NotImplementedError
+            If this operation is not supported by the model type.
+        """
+        pass
+
+    @abstractmethod
+    def compute_nli(self, premise: str, hypothesis: str) -> dict[str, float]:
+        """Compute natural language inference scores.
+
+        Must return dict with keys: "entailment", "neutral", "contradiction".
+        Required for inference-based constraints. Should raise NotImplementedError
+        if not supported by model type.
+
+        Parameters
+        ----------
+        premise : str
+            Premise text.
+        hypothesis : str
+            Hypothesis text.
+
+        Returns
+        -------
+        dict[str, float]
+            Dictionary with keys "entailment", "neutral", "contradiction"
+            mapping to probability scores that sum to ~1.0.
+
+        Raises
+        ------
+        NotImplementedError
+            If this operation is not supported by the model type.
+        """
+        pass
+
+    def compute_similarity(self, text1: str, text2: str) -> float:
+        """Compute similarity between two texts.
+
+        Default implementation using cosine similarity of embeddings.
+        Can be overridden for specialized similarity computation.
+
+        Parameters
+        ----------
+        text1 : str
+            First text.
+        text2 : str
+            Second text.
+
+        Returns
+        -------
+        float
+            Similarity score in [-1, 1] (cosine similarity).
+
+        Raises
+        ------
+        NotImplementedError
+            If embeddings are not supported by the model type.
+        """
+        emb1 = self.get_embedding(text1)
+        emb2 = self.get_embedding(text2)
+
+        # Cosine similarity
+        dot_product = np.dot(emb1, emb2)
+        norm1 = np.linalg.norm(emb1)
+        norm2 = np.linalg.norm(emb2)
+
+        if norm1 == 0 or norm2 == 0:
+            return 0.0
+
+        return float(dot_product / (norm1 * norm2))
+
+    def get_nli_label(self, premise: str, hypothesis: str) -> str:
+        """Get predicted NLI label (max score).
+
+        Default implementation using argmax over compute_nli() scores.
+
+        Parameters
+        ----------
+        premise : str
+            Premise text.
+        hypothesis : str
+            Hypothesis text.
+
+        Returns
+        -------
+        str
+            Predicted label: "entailment", "neutral", or "contradiction".
+
+        Raises
+        ------
+        NotImplementedError
+            If NLI is not supported by the model type.
+        """
+        scores = self.compute_nli(premise, hypothesis)
+        return max(scores, key=scores.get)  # type: ignore[arg-type, return-value]