evalring 0.2.0__py3-none-any.whl

EvalRing/config.py

@@ -0,0 +1,216 @@
+"""
+Provider configuration and credential resolution.
+
+EvalRing talks to any OpenAI-compatible chat-completions endpoint. Rather than
+hard-coding a vendor, every component resolves its credentials through
+:func:`resolve_credentials`, which reads a documented, ordered list of
+environment variables.
+
+See ``docs/CONFIGURATION.md`` for the full table and worked examples.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Final
+
+#: Base URL applied when a provider is selected but no explicit URL is given.
+#: ``None`` means "let the OpenAI SDK use its own default".
+_PROVIDER_DEFAULT_BASE_URL: Final[dict[str, str | None]] = {
+    "evalring": None,
+    "openai": None,
+    "openrouter": "https://openrouter.ai/api/v1",
+    "radium": "https://api.radium.cloud/v1",
+}
+
+#: Ordered credential sources: (provider, api-key var, base-url var).
+#: The first variable that is set wins. ``EVALRING_*`` is the vendor-neutral
+#: form and should be preferred by new users; the remaining entries exist so
+#: that existing provider-specific environments keep working unchanged.
+_CREDENTIAL_SOURCES: Final[tuple[tuple[str, str, str], ...]] = (
+    ("evalring", "EVALRING_API_KEY", "EVALRING_BASE_URL"),
+    ("openai", "OPENAI_API_KEY", "OPENAI_BASE_URL"),
+    ("openrouter", "OPENROUTER_API_KEY", "OPENROUTER_BASE_URL"),
+    ("openrouter", "OPEN_ROUTER_KEY", "OPENROUTER_BASE_URL"),
+    ("radium", "RADIUM_API_KEY", "RADIUM_BASE_URL"),
+)
+
+#: Ordered model-name variables, checked when no model is passed explicitly.
+_MODEL_VARS: Final[tuple[str, ...]] = (
+    "EVALRING_MODEL",
+    "OPENAI_MODEL",
+    "OPENROUTER_MODEL",
+    "OPEN_ROUTER_MODEL",
+    "RADIUM_MODEL",
+)
+
+#: Every variable EvalRing will read for an API key, in precedence order.
+CREDENTIAL_ENV_VARS: Final[tuple[str, ...]] = tuple(
+    key_var for _, key_var, _ in _CREDENTIAL_SOURCES
+)
+
+
+@dataclass(frozen=True)
+class ProviderCredentials:
+    """Resolved endpoint configuration for an OpenAI-compatible provider.
+
+    Attributes:
+        api_key: The API key, or ``None`` when nothing was configured.
+        base_url: The endpoint to call, or ``None`` to use the SDK default
+            (``https://api.openai.com/v1``).
+        provider: Short identifier of the source that supplied the key, one of
+            ``"evalring"``, ``"openai"``, ``"openrouter"``, ``"radium"``, or
+            ``"explicit"`` when the key was passed in code.
+        source: Human-readable description of where the key came from, safe to
+            print in diagnostics (never contains the key itself).
+    """
+
+    api_key: str | None
+    base_url: str | None
+    provider: str
+    source: str
+
+    def require_key(self) -> str:
+        """Return the API key, raising a descriptive error when absent.
+
+        Raises:
+            MissingCredentialsError: If no API key was resolved.
+        """
+        if not self.api_key:
+            raise MissingCredentialsError()
+        return self.api_key
+
+
+class MissingCredentialsError(ValueError):
+    """Raised when no API key could be resolved from arguments or environment."""
+
+    def __init__(self, message: str | None = None) -> None:
+        super().__init__(
+            message
+            or (
+                "No API key found. Pass api_key=... explicitly, or set one of: "
+                + ", ".join(CREDENTIAL_ENV_VARS)
+                + ". For a non-OpenAI endpoint also set EVALRING_BASE_URL. "
+                "See docs/CONFIGURATION.md."
+            )
+        )
+
+
+def _clean(value: str | None) -> str | None:
+    """Normalize an environment value, treating blank strings as unset."""
+    if value is None:
+        return None
+    stripped = value.strip()
+    return stripped or None
+
+
+def resolve_credentials(
+    api_key: str | None = None,
+    base_url: str | None = None,
+    *,
+    env: dict[str, str] | None = None,
+) -> ProviderCredentials:
+    """Resolve the API key and base URL for an OpenAI-compatible provider.
+
+    Explicit arguments always win. Otherwise the variables listed in
+    ``_CREDENTIAL_SOURCES`` are consulted in order; the first one that is set
+    determines both the key and, unless overridden, the base URL.
+
+    Args:
+        api_key: Explicit API key. When given, no key variable is consulted.
+        base_url: Explicit endpoint. When given, no URL variable is consulted.
+        env: Environment mapping to read from. Defaults to ``os.environ``;
+            supplying one makes the function trivially testable.
+
+    Returns:
+        A :class:`ProviderCredentials` describing the selected endpoint. The
+        ``api_key`` field may be ``None``; call
+        :meth:`ProviderCredentials.require_key` at the point of use so the
+        error message is raised where it is actionable.
+
+    Example:
+        >>> creds = resolve_credentials(env={"EVALRING_API_KEY": "sk-test"})
+        >>> creds.provider
+        'evalring'
+    """
+    environ = os.environ if env is None else env
+
+    explicit_key = _clean(api_key)
+    explicit_url = _clean(base_url)
+
+    if explicit_key:
+        return ProviderCredentials(
+            api_key=explicit_key,
+            base_url=explicit_url or _clean(environ.get("EVALRING_BASE_URL")),
+            provider="explicit",
+            source="api_key argument",
+        )
+
+    for provider, key_var, url_var in _CREDENTIAL_SOURCES:
+        key = _clean(environ.get(key_var))
+        if not key:
+            continue
+        resolved_url = (
+            explicit_url
+            or _clean(environ.get("EVALRING_BASE_URL"))
+            or _clean(environ.get(url_var))
+            or _PROVIDER_DEFAULT_BASE_URL.get(provider)
+        )
+        return ProviderCredentials(
+            api_key=key,
+            base_url=resolved_url,
+            provider=provider,
+            source=f"${key_var}",
+        )
+
+    return ProviderCredentials(
+        api_key=None,
+        base_url=explicit_url or _clean(environ.get("EVALRING_BASE_URL")),
+        provider="none",
+        source="unset",
+    )
+
+
+def resolve_model_name(
+    model_name: str | None = None,
+    *,
+    default: str | None = None,
+    env: dict[str, str] | None = None,
+) -> str | None:
+    """Resolve the model identifier to evaluate.
+
+    Args:
+        model_name: Explicit model identifier; returned unchanged when given.
+        default: Value returned when neither the argument nor any environment
+            variable is set.
+        env: Environment mapping to read from. Defaults to ``os.environ``.
+
+    Returns:
+        The resolved model identifier, or ``default`` when nothing is set.
+    """
+    environ = os.environ if env is None else env
+
+    explicit = _clean(model_name)
+    if explicit:
+        return explicit
+
+    for var in _MODEL_VARS:
+        value = _clean(environ.get(var))
+        if value:
+            return value
+
+    return default
+
+
+def has_any_credentials(env: dict[str, str] | None = None) -> bool:
+    """Report whether any recognized API-key variable is set.
+
+    Args:
+        env: Environment mapping to read from. Defaults to ``os.environ``.
+
+    Returns:
+        ``True`` if at least one credential variable holds a non-blank value.
+    """
+    environ = os.environ if env is None else env
+    return any(_clean(environ.get(var)) for var in CREDENTIAL_ENV_VARS)

EvalRing/dataset/__init__.py

@@ -0,0 +1,8 @@
+"""
+Dataset module for EvalRing.
+"""
+
+from .base import BaseDataset, DataSample
+from .implementations import CSVDataset, DataFrameDataset, JSONDataset
+
+__all__ = ["BaseDataset", "DataSample", "JSONDataset", "CSVDataset", "DataFrameDataset"]

EvalRing/dataset/ai_data_generator.py

@@ -0,0 +1,217 @@
+import asyncio
+import os
+from typing import Any, cast
+
+import pandas as pd
+from tqdm.asyncio import tqdm
+
+from ..config import resolve_credentials, resolve_model_name
+from ..logging_utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class AIDataGenerator:
+    """
+    A concurrent data generator that rewrites text instances from one class
+    label to another using an LLM, guided by a system prompt (typically the
+    class definitions).
+
+    Requires the optional ``datagen`` extra: ``pip install evalring[datagen]``.
+
+    Args:
+        api_key: API key. Resolved from the environment when omitted; see
+            :func:`EvalRing.config.resolve_credentials`.
+        model_name: Model identifier. Defaults to ``$EVALRING_MODEL``, then
+            ``"gpt-4o-mini"``.
+        base_url: OpenAI-compatible endpoint. Resolved alongside the API key
+            when omitted.
+        max_concurrent: Maximum number of in-flight requests.
+        system_prompt: System message prepended to every rewrite request.
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        model_name: str | None = None,
+        base_url: str | None = None,
+        max_concurrent: int = 50,
+        system_prompt: str = "",
+    ):
+        try:
+            from openai import AsyncOpenAI
+        except ImportError as exc:  # pragma: no cover - exercised via extras
+            raise ImportError(
+                "AIDataGenerator requires the 'openai' package. "
+                "Install it with: pip install evalring[datagen]"
+            ) from exc
+
+        credentials = resolve_credentials(api_key, base_url)
+        self.credentials = credentials
+        self.api_key = credentials.require_key()
+        self.base_url = credentials.base_url
+        self.model_name = resolve_model_name(model_name, default="gpt-4o-mini") or "gpt-4o-mini"
+        self.max_concurrent = max_concurrent
+        self.system_prompt = system_prompt
+
+        self.client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url)
+        self.semaphore = asyncio.Semaphore(self.max_concurrent)
+
+    async def _generate_single(
+        self, text: str, original_label: str, target_label: str, retries: int = 3
+    ) -> str:
+        """
+        Rewrites a single piece of text to fit the target_label.
+        """
+        async with self.semaphore:
+            prompt = (
+                f"You are an expert data generator and text rewriter.\n"
+                f"I will provide you with a user's text that is currently classified as '{original_label}'.\n"
+                f"Your task is to rewrite this text from the exact same user's perspective, but modifying it so that it "
+                f"naturally, realistically, and accurately fits the definition of the '{target_label}' class.\n\n"
+                f"CRITICAL INSTRUCTIONS:\n"
+                f"1. You MUST maintain the original first-person perspective of the poster.\n"
+                f"2. You MUST NOT write a reply, supportive message, or answer to the text.\n"
+                f"3. Do not make up your own class, strictly map the text to represent '{target_label}'.\n\n"
+                f"Original Text (Label: {original_label}):\n{text}\n\n"
+                f"Please provide ONLY the rewritten text for the '{target_label}' class. "
+                f"Do not include quotes, conversational filler, or markdown blocks. "
+                f"It should read like an authentic user post."
+            )
+
+            messages: list[Any] = []
+            if self.system_prompt:
+                messages.append({"role": "system", "content": self.system_prompt})
+
+            messages.append({"role": "user", "content": prompt})
+
+            for attempt in range(retries):
+                try:
+                    response = await self.client.chat.completions.create(
+                        model=self.model_name, messages=messages, temperature=0.7, max_tokens=1000
+                    )
+
+                    content = response.choices[0].message.content
+                    if content:
+                        return content.strip()
+                except Exception as e:
+                    if attempt == retries - 1:
+                        logger.warning(
+                            "Error generating data for text (target=%s): %s", target_label, e
+                        )
+                        return ""
+                    await asyncio.sleep(2**attempt)
+            return ""
+
+    async def generate_batch(
+        self, texts: list[str], current_labels: list[str], target_labels: list[str]
+    ) -> list[str]:
+        """
+        Concurrently process a batch of text rewrites.
+        """
+        tasks = [
+            self._generate_single(text, orig, tgt)
+            for text, orig, tgt in zip(texts, current_labels, target_labels, strict=False)
+        ]
+
+        # tqdm.gather provides a nice progress bar for the async tasks
+        results = await tqdm.gather(*tasks, desc="Generating data")
+        return results
+
+    def run_dataframe(
+        self,
+        df: pd.DataFrame,
+        text_col: str,
+        label_col: str,
+        target_col: str,
+        replace_original: bool = False,
+    ) -> pd.DataFrame:
+        """
+        Synchronous wrapper to process a Pandas DataFrame.
+        """
+        # Ensure we have an active event loop for the async execution
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        if loop.is_running():
+            import nest_asyncio
+
+            nest_asyncio.apply(loop)
+
+        texts = []
+        current_labels = []
+        target_labels = []
+        original_indices = []
+
+        is_list_of_classes = isinstance(target_col, (list, tuple, pd.Series))
+        target_col_list = list(target_col) if is_list_of_classes else None
+
+        for idx, row in df.iterrows():
+            text = row[text_col]
+            orig = row[label_col]
+
+            if is_list_of_classes:
+                # generate for all target classes except the original
+                targets = [t for t in (target_col_list or []) if t != orig]
+            else:
+                # generate specifically for the target_col specified in the row
+                targets = [row[target_col]]
+
+            for tgt in targets:
+                texts.append(text)
+                current_labels.append(orig)
+                target_labels.append(tgt)
+                original_indices.append(idx)
+
+        results = loop.run_until_complete(self.generate_batch(texts, current_labels, target_labels))
+
+        # Positional selection with a list of row numbers yields a DataFrame, but
+        # the pandas stubs describe iloc's return as a union that collapses to
+        # Series once numpy's own stubs are not in play.
+        df_out = cast(pd.DataFrame, df.iloc[list(original_indices)]).copy()
+
+        if replace_original:
+            df_out[text_col] = results
+            df_out[label_col] = target_labels
+            if not is_list_of_classes and isinstance(target_col, str):
+                if target_col in df_out.columns:
+                    df_out.drop(columns=[target_col], inplace=True)
+        else:
+            df_out["rewritten_text"] = results
+            df_out["target_label"] = target_labels
+
+        return df_out
+
+    def process_csv(
+        self,
+        input_csv_path: str,
+        text_col: str,
+        label_col: str,
+        target_col: str,
+        output_csv_path: str | None = None,
+    ) -> str:
+        """
+        Reads a CSV, generates new texts based on the target class, replaces the original
+        text and label fields, and saves to {original}_generated.csv.
+        """
+        if not output_csv_path:
+            base, ext = os.path.splitext(input_csv_path)
+            output_csv_path = f"{base}_generated{ext}"
+
+        df = pd.read_csv(input_csv_path)
+
+        logger.info("Loaded %d rows from %s", len(df), input_csv_path)
+        df_generated = self.run_dataframe(
+            df=df,
+            text_col=text_col,
+            label_col=label_col,
+            target_col=target_col,
+            replace_original=True,
+        )
+
+        df_generated.to_csv(output_csv_path, index=False)
+        logger.info("Saved generated dataset to %s", output_csv_path)
+        return output_csv_path

EvalRing/dataset/base.py

@@ -0,0 +1,238 @@
+"""
+Base classes for datasets in the EvalRing framework.
+"""
+
+import json
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+
+@dataclass
+class DataSample:
+    """Represents a single data sample for evaluation."""
+
+    id: str
+    input_text: str
+    target_output: Any
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary representation."""
+        return {
+            "id": self.id,
+            "input_text": self.input_text,
+            "target_output": self.target_output,
+            "metadata": self.metadata,
+        }
+
+
+class BaseDataset(ABC):
+    """
+    Abstract base class for all datasets in EvalRing.
+
+    This class provides a standardized interface for loading and accessing
+    different types of datasets (JSON, CSV, DataFrame, etc.).
+    """
+
+    def __init__(self, name: str, description: str | None = None, version: str = "1.0"):
+        self.name = name
+        self.description = description or f"Dataset: {name}"
+        self.version = version
+        self._samples: list[DataSample] = []
+        self._metadata: dict[str, Any] = {}
+
+    @abstractmethod
+    def load_data(
+        self,
+        source: str | Path | pd.DataFrame,
+        text_field: str = "text",
+        label_field: str = "label",
+        id_field: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Load samples from the given source.
+
+        Args:
+            source: File path or in-memory frame, depending on the subclass.
+            text_field: Column or key holding the model input.
+            label_field: Column or key holding the ground-truth label.
+            id_field: Column or key holding a unique per-sample ID. When
+                ``None`` the row index is used, which is only safe for datasets
+                that are never resumed or retried.
+            **kwargs: Reader-specific options (for example ``pandas.read_csv``
+                keyword arguments).
+        """
+        pass
+
+    @abstractmethod
+    def validate_data(self) -> bool:
+        """
+        Validate the loaded data for consistency and correctness.
+
+        Returns:
+            True if data is valid, False otherwise
+        """
+        pass
+
+    def assert_unique_ids(
+        self, *, expected_count: int | None = None, context: str = "dataset"
+    ) -> None:
+        """Raise if sample IDs are not unique.
+
+        Unique per-sample IDs are required for:
+        - resuming partial runs
+        - retrying failures
+        - merging per-sample metrics
+        """
+        ids = [str(s.id) for s in self._samples]
+        unique_ids = set(ids)
+
+        if expected_count is None:
+            expected_count = len(ids)
+
+        if len(ids) != expected_count:
+            raise ValueError(
+                f"[{context}] Expected {expected_count} samples, but found {len(ids)}."
+            )
+
+        if len(unique_ids) != len(ids):
+            # Provide a small, actionable duplicate preview.
+            from collections import Counter
+
+            counts = Counter(ids)
+            dupes = [k for k, v in counts.items() if v > 1]
+            preview = dupes[:10]
+            raise ValueError(
+                f"[{context}] Duplicate sample IDs detected: unique={len(unique_ids)} total={len(ids)} "
+                f"(first {len(preview)} duplicates: {preview}). "
+                "Fix by using a per-row question/message ID (e.g., an explicit 'ID' column or row index), not a user ID."
+            )
+
+    def add_sample(self, sample: DataSample) -> None:
+        """Add a single sample to the dataset."""
+        self._samples.append(sample)
+
+    def get_sample(self, index: int) -> DataSample:
+        """Get a sample by index."""
+        return self._samples[index]
+
+    def get_sample_by_id(self, sample_id: str) -> DataSample | None:
+        """Get a sample by its ID."""
+        for sample in self._samples:
+            if sample.id == sample_id:
+                return sample
+        return None
+
+    def __len__(self) -> int:
+        """Return the number of samples in the dataset."""
+        return len(self._samples)
+
+    def __iter__(self) -> Iterator[DataSample]:
+        """Iterate over samples in the dataset."""
+        return iter(self._samples)
+
+    def __getitem__(self, index: int | slice) -> DataSample | list[DataSample]:
+        """Get sample(s) by index or slice."""
+        return self._samples[index]
+
+    def get_statistics(self) -> dict[str, Any]:
+        """Get dataset statistics."""
+        if not self._samples:
+            return {"total_samples": 0}
+
+        # Basic statistics
+        stats = {
+            "total_samples": len(self._samples),
+            "name": self.name,
+            "version": self.version,
+            "description": self.description,
+        }
+
+        # Try to infer additional statistics
+        try:
+            input_lengths = [len(sample.input_text) for sample in self._samples]
+            stats.update(
+                {
+                    "avg_input_length": sum(input_lengths) / len(input_lengths),
+                    "min_input_length": min(input_lengths),
+                    "max_input_length": max(input_lengths),
+                }
+            )
+        except Exception:
+            pass
+
+        return stats
+
+    def split(
+        self, train_ratio: float = 0.8, seed: int | None = None
+    ) -> tuple["BaseDataset", "BaseDataset"]:
+        """
+        Split the dataset into train and test sets.
+
+        Args:
+            train_ratio: Ratio of data to use for training
+            seed: Random seed for reproducible splits
+
+        Returns:
+            Tuple of (train_dataset, test_dataset)
+        """
+        import random
+
+        if seed is not None:
+            random.seed(seed)
+
+        indices = list(range(len(self._samples)))
+        random.shuffle(indices)
+
+        split_idx = int(len(indices) * train_ratio)
+        train_indices = indices[:split_idx]
+        test_indices = indices[split_idx:]
+
+        train_dataset = self.__class__(name=f"{self.name}_train", version=self.version)
+        test_dataset = self.__class__(name=f"{self.name}_test", version=self.version)
+
+        for idx in train_indices:
+            train_dataset.add_sample(self._samples[idx])
+
+        for idx in test_indices:
+            test_dataset.add_sample(self._samples[idx])
+
+        return train_dataset, test_dataset
+
+    def save(self, filepath: str | Path) -> None:
+        """Save dataset to file."""
+        data = {
+            "name": self.name,
+            "description": self.description,
+            "version": self.version,
+            "metadata": self._metadata,
+            "samples": [sample.to_dict() for sample in self._samples],
+        }
+
+        with open(filepath, "w") as f:
+            json.dump(data, f, indent=2)
+
+    def load_from_file(self, filepath: str | Path) -> None:
+        """Load dataset from file."""
+        with open(filepath) as f:
+            data = json.load(f)
+
+        self.name = data.get("name", self.name)
+        self.description = data.get("description", self.description)
+        self.version = data.get("version", self.version)
+        self._metadata = data.get("metadata", {})
+
+        self._samples = []
+        for sample_data in data.get("samples", []):
+            sample = DataSample(
+                id=sample_data["id"],
+                input_text=sample_data["input_text"],
+                target_output=sample_data["target_output"],
+                metadata=sample_data.get("metadata", {}),
+            )
+            self._samples.append(sample)