themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/backends/storage.py

@@ -0,0 +1,260 @@
+"""Storage backend interface for custom storage implementations.
+
+This module defines the abstract interface for storage backends, allowing
+users to implement custom storage solutions (cloud storage, databases, etc.)
+without modifying Themis core code.
+
+Example implementations:
+- S3Backend: Store results in AWS S3
+- GCSBackend: Store results in Google Cloud Storage
+- PostgresBackend: Store results in PostgreSQL
+- RedisBackend: Use Redis for distributed caching
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List
+
+from themis.core.entities import (
+    EvaluationRecord,
+    ExperimentReport,
+    GenerationRecord,
+)
+
+
+class StorageBackend(ABC):
+    """Abstract interface for storage backends.
+    
+    Implement this interface to create custom storage solutions.
+    All methods should be thread-safe if used with concurrent workers.
+    
+    Example:
+        >>> class S3StorageBackend(StorageBackend):
+        ...     def __init__(self, bucket: str):
+        ...         self.bucket = bucket
+        ...         self.s3_client = boto3.client('s3')
+        ...
+        ...     def save_run_metadata(self, run_id: str, metadata: RunMetadata) -> None:
+        ...         key = f"runs/{run_id}/metadata.json"
+        ...         self.s3_client.put_object(
+        ...             Bucket=self.bucket,
+        ...             Key=key,
+        ...             Body=metadata.to_json(),
+        ...         )
+        ...     # ... implement other methods
+    """
+    
+    @abstractmethod
+    def save_run_metadata(self, run_id: str, metadata: Dict[str, Any]) -> None:
+        """Save run metadata.
+        
+        Args:
+            run_id: Unique identifier for the run
+            metadata: Run metadata to save (as dictionary)
+        """
+        pass
+    
+    @abstractmethod
+    def load_run_metadata(self, run_id: str) -> Dict[str, Any]:
+        """Load run metadata.
+        
+        Args:
+            run_id: Unique identifier for the run
+            
+        Returns:
+            Run metadata as dictionary
+            
+        Raises:
+            FileNotFoundError: If run metadata doesn't exist
+        """
+        pass
+    
+    @abstractmethod
+    def save_generation_record(self, run_id: str, record: GenerationRecord) -> None:
+        """Save a generation record.
+        
+        Args:
+            run_id: Unique identifier for the run
+            record: Generation record to save
+            
+        Note:
+            This method should be atomic and thread-safe.
+        """
+        pass
+    
+    @abstractmethod
+    def load_generation_records(self, run_id: str) -> List[GenerationRecord]:
+        """Load all generation records for a run.
+        
+        Args:
+            run_id: Unique identifier for the run
+            
+        Returns:
+            List of generation records
+        """
+        pass
+    
+    @abstractmethod
+    def save_evaluation_record(self, run_id: str, record: EvaluationRecord) -> None:
+        """Save an evaluation record.
+        
+        Args:
+            run_id: Unique identifier for the run
+            record: Evaluation record to save
+            
+        Note:
+            This method should be atomic and thread-safe.
+        """
+        pass
+    
+    @abstractmethod
+    def load_evaluation_records(self, run_id: str) -> Dict[str, EvaluationRecord]:
+        """Load all evaluation records for a run.
+        
+        Args:
+            run_id: Unique identifier for the run
+            
+        Returns:
+            Dictionary mapping cache_key to EvaluationRecord
+        """
+        pass
+    
+    @abstractmethod
+    def save_report(self, run_id: str, report: ExperimentReport) -> None:
+        """Save experiment report.
+        
+        Args:
+            run_id: Unique identifier for the run
+            report: Experiment report to save
+        """
+        pass
+    
+    @abstractmethod
+    def load_report(self, run_id: str) -> ExperimentReport:
+        """Load experiment report.
+        
+        Args:
+            run_id: Unique identifier for the run
+            
+        Returns:
+            Experiment report
+            
+        Raises:
+            FileNotFoundError: If report doesn't exist
+        """
+        pass
+    
+    @abstractmethod
+    def list_runs(self) -> List[str]:
+        """List all run IDs in storage.
+        
+        Returns:
+            List of run IDs
+        """
+        pass
+    
+    @abstractmethod
+    def run_exists(self, run_id: str) -> bool:
+        """Check if a run exists in storage.
+        
+        Args:
+            run_id: Unique identifier for the run
+            
+        Returns:
+            True if run exists, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def delete_run(self, run_id: str) -> None:
+        """Delete all data for a run.
+        
+        Args:
+            run_id: Unique identifier for the run
+        """
+        pass
+    
+    def close(self) -> None:
+        """Close the storage backend and release resources.
+        
+        Optional method for cleanup. Called when storage is no longer needed.
+        """
+        pass
+
+
+class LocalFileStorageBackend(StorageBackend):
+    """Adapter for the existing ExperimentStorage implementation.
+    
+    This class wraps the current file-based storage implementation
+    to conform to the StorageBackend interface.
+    
+    Note:
+        This is a compatibility layer. New code should use the interface,
+        but existing storage logic is preserved.
+    """
+    
+    def __init__(self, storage_path: str | Path):
+        """Initialize with path to storage directory.
+        
+        Args:
+            storage_path: Path to storage directory
+        """
+        from themis.experiment.storage import ExperimentStorage
+        self._storage = ExperimentStorage(storage_path)
+    
+    def save_run_metadata(self, run_id: str, metadata: Dict[str, Any]) -> None:
+        """Save run metadata."""
+        experiment_id = metadata.get("experiment_id", "default")
+        self._storage.start_run(run_id, experiment_id=experiment_id)
+    
+    def load_run_metadata(self, run_id: str) -> Dict[str, Any]:
+        """Load run metadata."""
+        # Note: Current storage doesn't have a direct method for this
+        # This is a limitation of the adapter pattern
+        raise NotImplementedError("Use ExperimentStorage directly for now")
+    
+    def save_generation_record(self, run_id: str, record: GenerationRecord) -> None:
+        """Save generation record."""
+        self._storage.append_record(run_id, record)
+    
+    def load_generation_records(self, run_id: str) -> List[GenerationRecord]:
+        """Load generation records."""
+        cached = self._storage.load_cached_records(run_id)
+        return list(cached.values())
+    
+    def save_evaluation_record(self, run_id: str, record: EvaluationRecord) -> None:
+        """Save evaluation record."""
+        self._storage.append_evaluation(run_id, record)
+    
+    def load_evaluation_records(self, run_id: str) -> Dict[str, EvaluationRecord]:
+        """Load evaluation records."""
+        return self._storage.load_cached_evaluations(run_id)
+    
+    def save_report(self, run_id: str, report: ExperimentReport) -> None:
+        """Save report."""
+        self._storage.save_report(run_id, report)
+    
+    def load_report(self, run_id: str) -> ExperimentReport:
+        """Load report."""
+        return self._storage.load_report(run_id)
+    
+    def list_runs(self) -> List[str]:
+        """List runs."""
+        return self._storage.list_runs()
+    
+    def run_exists(self, run_id: str) -> bool:
+        """Check if run exists."""
+        return run_id in self._storage.list_runs()
+    
+    def delete_run(self, run_id: str) -> None:
+        """Delete run."""
+        # Note: Current storage doesn't have delete functionality
+        raise NotImplementedError("Delete not implemented in current storage")
+
+
+__all__ = [
+    "StorageBackend",
+    "LocalFileStorageBackend",
+]

themis/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Command-line helpers for running Themis experiments."""
+
+from . import main
+
+__all__ = ["main"]

themis/cli/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for running themis.cli as a module."""
+
+from .main import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

themis/cli/commands/__init__.py

@@ -0,0 +1,19 @@
+"""CLI command modules."""
+
+from themis.cli.commands import (
+    benchmarks,
+    config_commands,
+    demo,
+    info,
+    math_benchmarks,
+    mcq_benchmarks,
+)
+
+__all__ = [
+    "benchmarks",
+    "config_commands",
+    "demo",
+    "info",
+    "math_benchmarks",
+    "mcq_benchmarks",
+]

themis/cli/commands/benchmarks.py

@@ -0,0 +1,221 @@
+"""Benchmark listing commands."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.providers.registry import _REGISTRY
+
+
+def list_providers(
+    *,
+    verbose: Annotated[
+        bool, Parameter(help="Show detailed provider information")
+    ] = False,
+) -> int:
+    """List available LLM providers."""
+    providers = sorted(_REGISTRY._factories.keys())
+
+    if not providers:
+        print("No providers registered.")
+        return 0
+
+    print("Available Providers:")
+    print("=" * 60)
+
+    provider_info = {
+        "fake": "Built-in fake provider for testing (no API required)",
+        "openai-compatible": "OpenAI-compatible API (LM Studio, Ollama, vLLM, OpenAI)",
+        "vllm": "vLLM server provider for local model hosting",
+    }
+
+    for provider in providers:
+        status = "✓" if provider in provider_info else "·"
+        print(f"{status} {provider}")
+        if verbose and provider in provider_info:
+            print(f"  {provider_info[provider]}")
+
+    if not verbose:
+        print("\nUse --verbose for more details")
+
+    return 0
+
+
+def list_benchmarks(
+    *,
+    verbose: Annotated[
+        bool, Parameter(help="Show detailed benchmark information")
+    ] = False,
+) -> int:
+    """List available datasets and benchmarks."""
+    benchmarks = [
+        {
+            "name": "math500",
+            "description": "MATH-500 dataset for mathematical reasoning",
+            "source": "huggingface (default) or local",
+            "subjects": [
+                "algebra",
+                "counting_and_probability",
+                "geometry",
+                "intermediate_algebra",
+                "number_theory",
+                "prealgebra",
+                "precalculus",
+            ],
+            "command": "uv run python -m themis.cli math500",
+        },
+        {
+            "name": "gsm8k",
+            "description": "GSM8K dataset for grade school math word problems",
+            "source": "huggingface (default) or local",
+            "subjects": "math",
+            "command": "uv run python -m themis.cli gsm8k",
+        },
+        {
+            "name": "gpqa",
+            "description": "GPQA dataset for graduate-level science questions",
+            "source": "huggingface (default) or local",
+            "subjects": "science",
+            "command": "uv run python -m themis.cli gpqa",
+        },
+        {
+            "name": "gsm-symbolic",
+            "description": "GSM-Symbolic dataset for symbolic math reasoning",
+            "source": "huggingface (default) or local",
+            "subjects": "math",
+            "command": "uv run python -m themis.cli gsm-symbolic",
+        },
+        {
+            "name": "medmcqa",
+            "description": "MedMCQA dataset for medical entrance exams",
+            "source": "huggingface (default) or local",
+            "subjects": "medicine",
+            "command": "uv run python -m themis.cli medmcqa",
+        },
+        {
+            "name": "med_qa",
+            "description": "MedQA dataset for medical question answering",
+            "source": "huggingface (default) or local",
+            "subjects": "medicine",
+            "command": "uv run python -m themis.cli med_qa",
+        },
+        {
+            "name": "sciq",
+            "description": "SciQ dataset for science questions",
+            "source": "huggingface (default) or local",
+            "subjects": "science",
+            "command": "uv run python -m themis.cli sciq",
+        },
+        {
+            "name": "commonsense_qa",
+            "description": "CommonsenseQA dataset for commonsense reasoning",
+            "source": "huggingface (default) or local",
+            "subjects": "commonsense",
+            "command": "uv run python -m themis.cli commonsense_qa",
+        },
+        {
+            "name": "piqa",
+            "description": "PIQA dataset for physical commonsense reasoning",
+            "source": "huggingface (default) or local",
+            "subjects": "commonsense",
+            "command": "uv run python -m themis.cli piqa",
+        },
+        {
+            "name": "social_i_qa",
+            "description": "Social IQA dataset for social commonsense reasoning",
+            "source": "huggingface (default) or local",
+            "subjects": "commonsense",
+            "command": "uv run python -m themis.cli social_i_qa",
+        },
+        {
+            "name": "coqa",
+            "description": "CoQA dataset for conversational question answering",
+            "source": "huggingface (default) or local",
+            "subjects": "conversational",
+            "command": "uv run python -m themis.cli coqa",
+        },
+        {
+            "name": "supergpqa",
+            "description": "Graduate-level QA benchmark with multiple-choice questions",
+            "source": "huggingface (default) or local",
+            "subjects": "category filter via --subjects",
+            "command": "uv run python -m themis.cli supergpqa",
+        },
+        {
+            "name": "mmlu-pro",
+            "description": "Professional-level MMLU benchmark with refined distractors",
+            "source": "huggingface (default) or local",
+            "subjects": "subject filter via --subjects",
+            "command": "uv run python -m themis.cli mmlu-pro",
+        },
+        {
+            "name": "aime24",
+            "description": "AIME 2024 competition problems",
+            "source": "huggingface (default) or local",
+            "subjects": "problem set",
+            "command": "uv run python -m themis.cli aime24",
+        },
+        {
+            "name": "aime25",
+            "description": "AIME 2025 competition problems",
+            "source": "huggingface (default) or local",
+            "subjects": "problem set",
+            "command": "uv run python -m themis.cli aime25",
+        },
+        {
+            "name": "amc23",
+            "description": "AMC 2023 competition problems",
+            "source": "huggingface (default) or local",
+            "subjects": "problem set",
+            "command": "uv run python -m themis.cli amc23",
+        },
+        {
+            "name": "olympiadbench",
+            "description": "Mixed Olympiad-style math benchmark",
+            "source": "huggingface (default) or local",
+            "subjects": "competition metadata",
+            "command": "uv run python -m themis.cli olympiadbench",
+        },
+        {
+            "name": "beyondaime",
+            "description": "BeyondAIME advanced math competition set",
+            "source": "huggingface (default) or local",
+            "subjects": "problem set",
+            "command": "uv run python -m themis.cli beyondaime",
+        },
+        {
+            "name": "demo",
+            "description": "Built-in demo with 2 math problems",
+            "source": "inline",
+            "subjects": ["precalculus", "arithmetic"],
+            "command": "uv run python -m themis.cli demo",
+        },
+        {
+            "name": "inline",
+            "description": "Custom inline dataset (via config file)",
+            "source": "config file",
+            "subjects": "user-defined",
+            "command": "uv run python -m themis.cli run-config --config your_config.yaml",
+        },
+    ]
+
+    print("Available Datasets & Benchmarks:")
+    print("=" * 60)
+
+    for bench in benchmarks:
+        print(f"\n📊 {bench['name']}")
+        print(f"   {bench['description']}")
+        if verbose:
+            print(f"   Source: {bench['source']}")
+            if isinstance(bench["subjects"], list):
+                print(f"   Subjects: {', '.join(bench['subjects'])}")
+            else:
+                print(f"   Subjects: {bench['subjects']}")
+            print(f"   Command: {bench['command']}")
+
+    if not verbose:
+        print("\nUse --verbose for more details and example commands")
+
+    return 0