themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/presets/models.py

@@ -0,0 +1,190 @@
+"""Model name parsing and provider detection.
+
+This module automatically detects the appropriate provider based on
+model names, eliminating the need for users to specify providers manually.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def parse_model_name(model: str, **kwargs: Any) -> tuple[str, str, dict[str, Any]]:
+    """Parse model name and detect provider.
+    
+    Args:
+        model: Model identifier (e.g., "gpt-4", "claude-3-opus", "llama-2-70b")
+        **kwargs: Additional provider-specific options
+    
+    Returns:
+        Tuple of (provider_name, model_id, provider_options)
+    
+    Examples:
+        >>> parse_model_name("gpt-4")
+        ("litellm", "gpt-4", {})
+        
+        >>> parse_model_name("claude-3-opus-20240229")
+        ("litellm", "claude-3-opus-20240229", {})
+        
+        >>> parse_model_name("local-llm", base_url="http://localhost:1234/v1")
+        ("litellm", "local-llm", {"base_url": "http://localhost:1234/v1"})
+    """
+    model_lower = model.lower()
+    
+    # OpenAI models
+    if any(pattern in model_lower for pattern in ["gpt-", "o1-", "text-davinci"]):
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # Anthropic models
+    if "claude" in model_lower:
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # Google models
+    if any(pattern in model_lower for pattern in ["gemini", "palm"]):
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # Meta models
+    if "llama" in model_lower:
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # Mistral models
+    if "mistral" in model_lower or "mixtral" in model_lower:
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # Cohere models
+    if "command" in model_lower and "xl" in model_lower:
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # AI21 models
+    if "j2-" in model_lower:
+        return "litellm", model, _extract_provider_options(kwargs)
+    
+    # Fake model for testing
+    if "fake" in model_lower:
+        return "fake", model, {}
+    
+    # Default: assume it's a litellm-compatible model
+    # User can provide base_url for custom endpoints
+    return "litellm", model, _extract_provider_options(kwargs)
+
+
+def _extract_provider_options(kwargs: dict[str, Any]) -> dict[str, Any]:
+    """Extract provider-specific options from kwargs.
+    
+    Args:
+        kwargs: Dictionary of options
+    
+    Returns:
+        Dictionary of provider options
+    """
+    provider_options = {}
+    
+    # Known provider options
+    option_keys = [
+        "api_key",
+        "base_url",
+        "api_base",
+        "api_version",
+        "timeout",
+        "max_retries",
+        "n_parallel",
+        "organization",
+        "api_type",
+        "region_name",
+    ]
+    
+    for key in option_keys:
+        if key in kwargs:
+            provider_options[key] = kwargs[key]
+    
+    return provider_options
+
+
+def get_provider_for_model(model: str) -> str:
+    """Get provider name for a model (without parsing full options).
+    
+    Args:
+        model: Model identifier
+    
+    Returns:
+        Provider name
+    
+    Examples:
+        >>> get_provider_for_model("gpt-4")
+        "litellm"
+        
+        >>> get_provider_for_model("claude-3-opus")
+        "litellm"
+    """
+    provider, _, _ = parse_model_name(model)
+    return provider
+
+
+# Model family detection for preset selection
+def get_model_family(model: str) -> str:
+    """Get the model family for capability detection.
+    
+    Args:
+        model: Model identifier
+    
+    Returns:
+        Model family name
+    
+    Examples:
+        >>> get_model_family("gpt-4-turbo")
+        "gpt-4"
+        
+        >>> get_model_family("claude-3-opus-20240229")
+        "claude-3"
+    """
+    model_lower = model.lower()
+    
+    # OpenAI families
+    if "gpt-4" in model_lower:
+        return "gpt-4"
+    if "gpt-3.5" in model_lower:
+        return "gpt-3.5"
+    if "o1" in model_lower:
+        return "o1"
+    
+    # Anthropic families
+    if "claude-3" in model_lower:
+        if "opus" in model_lower:
+            return "claude-3-opus"
+        elif "sonnet" in model_lower:
+            return "claude-3-sonnet"
+        elif "haiku" in model_lower:
+            return "claude-3-haiku"
+        return "claude-3"
+    if "claude-2" in model_lower:
+        return "claude-2"
+    
+    # Google families
+    if "gemini-pro" in model_lower:
+        return "gemini-pro"
+    if "gemini-ultra" in model_lower:
+        return "gemini-ultra"
+    
+    # Meta families
+    if "llama-2" in model_lower:
+        if "70b" in model_lower:
+            return "llama-2-70b"
+        elif "13b" in model_lower:
+            return "llama-2-13b"
+        elif "7b" in model_lower:
+            return "llama-2-7b"
+        return "llama-2"
+    if "llama-3" in model_lower:
+        return "llama-3"
+    
+    # Mistral families
+    if "mixtral" in model_lower:
+        return "mixtral"
+    if "mistral" in model_lower:
+        return "mistral"
+    
+    return "unknown"
+
+
+__all__ = ["parse_model_name", "get_provider_for_model", "get_model_family"]

themis/project/__init__.py

@@ -0,0 +1,20 @@
+"""Project helpers for managing experiment collections."""
+
+from themis.project.definitions import Project, ProjectExperiment
+from themis.project.patterns import (
+    AblationChart,
+    AblationChartPoint,
+    AblationVariant,
+    XAbationPattern,
+    XAbationPatternApplication,
+)
+
+__all__ = [
+    "Project",
+    "ProjectExperiment",
+    "AblationChart",
+    "AblationChartPoint",
+    "AblationVariant",
+    "XAbationPattern",
+    "XAbationPatternApplication",
+]

themis/project/definitions.py

@@ -0,0 +1,98 @@
+"""Project-level definitions for grouping experiments."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Mapping, Sequence
+
+from themis.experiment.definitions import ExperimentDefinition
+
+
+@dataclass(frozen=True)
+class ProjectExperiment:
+    """Metadata wrapper that pairs a name with an experiment definition."""
+
+    name: str
+    definition: ExperimentDefinition
+    description: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+
+@dataclass
+class Project:
+    """Container that organizes multiple experiments under a shared project."""
+
+    project_id: str
+    name: str
+    description: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    tags: tuple[str, ...] = field(default_factory=tuple)
+    experiments: Sequence[ProjectExperiment] = field(default_factory=tuple)
+
+    def __post_init__(self) -> None:
+        self._experiment_index: dict[str, ProjectExperiment] = {}
+        normalized: list[ProjectExperiment] = []
+        for experiment in self.experiments:
+            self._register_experiment(experiment)
+            normalized.append(experiment)
+        self.experiments = tuple(normalized)
+
+    def add_experiment(self, experiment: ProjectExperiment) -> ProjectExperiment:
+        """Attach an experiment to the project, enforcing unique names."""
+
+        self._register_experiment(experiment)
+        self.experiments = tuple(list(self.experiments) + [experiment])
+        return experiment
+
+    def create_experiment(
+        self,
+        *,
+        name: str,
+        definition: ExperimentDefinition,
+        description: str | None = None,
+        metadata: Mapping[str, Any] | None = None,
+        tags: Sequence[str] | None = None,
+    ) -> ProjectExperiment:
+        """Convenience helper to register an experiment from raw components."""
+
+        experiment = ProjectExperiment(
+            name=name,
+            description=description,
+            definition=definition,
+            metadata=dict(metadata or {}),
+            tags=tuple(tags or ()),
+        )
+        return self.add_experiment(experiment)
+
+    def get_experiment(self, name: str) -> ProjectExperiment:
+        try:
+            return self._experiment_index[name]
+        except KeyError as exc:  # pragma: no cover - defensive guard
+            raise KeyError(
+                f"Experiment '{name}' not registered in project '{self.project_id}'"
+            ) from exc
+
+    def metadata_for_experiment(self, name: str) -> dict[str, Any]:
+        """Merge project-level metadata with experiment-specific overrides."""
+
+        combined = dict(self.metadata)
+        combined.update(self.get_experiment(name).metadata)
+        return combined
+
+    def list_experiment_names(self) -> tuple[str, ...]:
+        return tuple(self._experiment_index.keys())
+
+    def _register_experiment(self, experiment: ProjectExperiment) -> None:
+        if experiment.name in self._experiment_index:
+            raise ValueError(
+                f"Experiment '{experiment.name}' already registered "
+                f"in project '{self.project_id}'"
+            )
+        self._experiment_index[experiment.name] = experiment
+
+
+__all__ = [
+    "Project",
+    "ProjectExperiment",
+]

themis/project/patterns.py

@@ -0,0 +1,230 @@
+"""Reusable experiment patterns for organizing projects."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any, Callable, Mapping, Sequence
+
+from themis.experiment.definitions import ExperimentDefinition
+from themis.experiment.orchestrator import ExperimentReport
+from themis.project.definitions import Project, ProjectExperiment
+
+
+def _slugify(value: str) -> str:
+    text = value.strip().lower()
+    text = re.sub(r"[^a-z0-9]+", "-", text)
+    text = text.strip("-")
+    return text or "variant"
+
+
+@dataclass
+class AblationVariant:
+    value: Any
+    label: str | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def display_label(self) -> str:
+        return self.label or str(self.value)
+
+    def slug(self) -> str:
+        return _slugify(self.display_label())
+
+
+@dataclass(frozen=True)
+class AblationChartPoint:
+    x_value: Any
+    label: str
+    metric_value: float
+    metric_name: str
+    count: int
+
+
+@dataclass(frozen=True)
+class AblationChart:
+    title: str
+    x_label: str
+    y_label: str
+    metric_name: str
+    points: tuple[AblationChartPoint, ...]
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "title": self.title,
+            "x_label": self.x_label,
+            "y_label": self.y_label,
+            "metric": self.metric_name,
+            "points": [
+                {
+                    "label": point.label,
+                    "x": point.x_value,
+                    "value": point.metric_value,
+                    "count": point.count,
+                }
+                for point in self.points
+            ],
+        }
+
+
+@dataclass(frozen=True)
+class XAbationPatternApplication:
+    pattern_name: str
+    parameter_name: str
+    experiments: tuple[ProjectExperiment, ...]
+    variant_by_name: Mapping[str, AblationVariant]
+    _pattern: "XAblationPattern" = field(repr=False)
+
+    def build_chart(self, reports: Mapping[str, ExperimentReport]) -> AblationChart:
+        return self._pattern._build_chart(reports, self.variant_by_name)
+
+
+class XAbationPattern:
+    """Vary a single factor across values to compare performance."""
+
+    pattern_type = "x-ablation"
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        parameter_name: str,
+        values: Sequence[AblationVariant | Any],
+        definition_builder: Callable[[AblationVariant], ExperimentDefinition],
+        metric_name: str,
+        x_axis_label: str | None = None,
+        y_axis_label: str | None = None,
+        title: str | None = None,
+    ) -> None:
+        if not values:
+            raise ValueError("XAblationPattern requires at least one value")
+        self.name = name
+        self.parameter_name = parameter_name
+        self._variants = [self._normalize_variant(value) for value in values]
+        self._definition_builder = definition_builder
+        self.metric_name = metric_name
+        self.x_axis_label = x_axis_label or parameter_name
+        self.y_axis_label = y_axis_label or metric_name
+        self.title = title or f"{name} ({parameter_name} ablation)"
+
+    def materialize(
+        self,
+        project: Project,
+        *,
+        name_template: str | None = None,
+        description_template: str | None = None,
+        base_tags: Sequence[str] | None = None,
+    ) -> XAbationPatternApplication:
+        template = name_template or "{pattern}-{value_slug}"
+        tags = tuple(base_tags or ()) + (self.pattern_type,)
+        experiments: list[ProjectExperiment] = []
+        variant_map: dict[str, AblationVariant] = {}
+        for index, variant in enumerate(self._variants):
+            experiment_name = template.format(
+                pattern=self.name,
+                parameter=self.parameter_name,
+                value=variant.value,
+                value_label=variant.display_label(),
+                value_slug=variant.slug(),
+                index=index,
+            )
+            description: str | None = None
+            if description_template is not None:
+                description = description_template.format(
+                    pattern=self.name,
+                    parameter=self.parameter_name,
+                    value=variant.value,
+                    value_label=variant.display_label(),
+                    index=index,
+                )
+            metadata = {
+                "pattern": self.pattern_type,
+                "pattern_name": self.name,
+                "parameter_name": self.parameter_name,
+                "parameter_value": variant.value,
+                "parameter_label": variant.display_label(),
+                "pattern_index": index,
+            }
+            metadata.update(dict(variant.metadata))
+            definition = self._definition_builder(variant)
+            project_experiment = project.add_experiment(
+                ProjectExperiment(
+                    name=experiment_name,
+                    description=description,
+                    definition=definition,
+                    metadata=metadata,
+                    tags=tuple(dict.fromkeys(tags)),
+                )
+            )
+            experiments.append(project_experiment)
+            variant_map[project_experiment.name] = variant
+        return XAbationPatternApplication(
+            pattern_name=self.name,
+            parameter_name=self.parameter_name,
+            experiments=tuple(experiments),
+            variant_by_name=variant_map,
+            _pattern=self,
+        )
+
+    def _build_chart(
+        self,
+        reports: Mapping[str, ExperimentReport],
+        variant_by_name: Mapping[str, AblationVariant],
+    ) -> AblationChart:
+        points: list[AblationChartPoint] = []
+        for experiment in variant_by_name:
+            variant = variant_by_name[experiment]
+            report = reports.get(experiment)
+            if report is None:
+                raise KeyError(
+                    f"Missing report for experiment '{experiment}' in pattern '{self.name}'"
+                )
+            metric = report.evaluation_report.metrics.get(self.metric_name)
+            if metric is None:
+                raise ValueError(
+                    f"Metric '{self.metric_name}' not found for experiment '{experiment}'"
+                )
+            points.append(
+                AblationChartPoint(
+                    x_value=variant.value,
+                    label=variant.display_label(),
+                    metric_value=metric.mean,
+                    metric_name=metric.name,
+                    count=metric.count,
+                )
+            )
+        ordered_points = self._order_points(points, variant_by_name)
+        return AblationChart(
+            title=self.title,
+            x_label=self.x_axis_label,
+            y_label=self.y_axis_label,
+            metric_name=self.metric_name,
+            points=tuple(ordered_points),
+        )
+
+    def _order_points(
+        self,
+        points: Sequence[AblationChartPoint],
+        variant_by_name: Mapping[str, AblationVariant],
+    ) -> list[AblationChartPoint]:
+        order: dict[Any, int] = {
+            variant.value: index for index, variant in enumerate(self._variants)
+        }
+        return sorted(points, key=lambda point: order.get(point.x_value, 0))
+
+    def _normalize_variant(self, value: AblationVariant | Any) -> AblationVariant:
+        if isinstance(value, AblationVariant):
+            return AblationVariant(
+                value=value.value,
+                label=value.label,
+                metadata=dict(value.metadata),
+            )
+        return AblationVariant(value=value)
+
+
+__all__ = [
+    "AblationChart",
+    "AblationChartPoint",
+    "AblationVariant",
+    "XAblationPattern",
+    "XAblationPatternApplication",
+]

themis/providers/__init__.py

@@ -0,0 +1,5 @@
+"""Model provider registry and helpers."""
+
+from .registry import ProviderFactory, create_provider, register_provider
+
+__all__ = ["register_provider", "create_provider", "ProviderFactory"]

themis/providers/registry.py

@@ -0,0 +1,39 @@
+"""Simple registry for ModelProvider factories."""
+
+from __future__ import annotations
+
+from typing import Callable, Dict
+
+from themis.interfaces import ModelProvider
+
+ProviderFactory = Callable[..., ModelProvider]
+
+
+class _ProviderRegistry:
+    def __init__(self) -> None:
+        self._factories: Dict[str, ProviderFactory] = {}
+
+    def register(self, name: str, factory: ProviderFactory) -> None:
+        key = name.lower()
+        self._factories[key] = factory
+
+    def create(self, name: str, **options) -> ModelProvider:
+        key = name.lower()
+        factory = self._factories.get(key)
+        if factory is None:
+            raise KeyError(f"No provider registered under name '{name}'")
+        return factory(**options)
+
+
+_REGISTRY = _ProviderRegistry()
+
+
+def register_provider(name: str, factory: ProviderFactory) -> None:
+    _REGISTRY.register(name, factory)
+
+
+def create_provider(name: str, **options) -> ModelProvider:
+    return _REGISTRY.create(name, **options)
+
+
+__all__ = ["register_provider", "create_provider", "ProviderFactory"]

themis/server/__init__.py

@@ -0,0 +1,28 @@
+"""FastAPI server for Themis web dashboard.
+
+This module provides a REST API and WebSocket interface for:
+- Listing and viewing experiment runs
+- Comparing multiple runs
+- Real-time monitoring of running experiments
+- Exporting results in various formats
+
+The server is optional and requires the 'server' extra:
+    pip install themis[server]
+    # or
+    uv pip install themis[server]
+
+Usage:
+    # Start the server
+    themis serve --port 8080
+    
+    # Or programmatically
+    from themis.server import create_app
+    app = create_app(storage_path=".cache/experiments")
+    
+    # Run with uvicorn
+    uvicorn themis.server:app --host 0.0.0.0 --port 8080
+"""
+
+from themis.server.app import create_app
+
+__all__ = ["create_app"]