themis-eval 0.2.3__py3-none-any.whl → 1.0.0__py3-none-any.whl

themis/__init__.py

@@ -12,9 +12,10 @@ Extension APIs for registering custom components:
     - themis.register_benchmark() - Register custom benchmark presets
 """
 
-from themis import config, core, evaluation, experiment, generation, project
+from themis import config, core, evaluation, generation, project, session
 from themis._version import __version__
 from themis.api import evaluate, get_registered_metrics, register_metric
+from themis.session import ExperimentSession
 from themis.datasets import register_dataset, list_datasets, is_dataset_registered
 from themis.presets import register_benchmark, list_benchmarks, get_benchmark_preset
 from themis.providers import register_provider
@@ -39,9 +40,11 @@ __all__ = [
     "config",
     "core",
     "evaluation",
-    "experiment",
     "generation",
     "project",
+    "session",
+    # Session API
+    "ExperimentSession",
     # Version
     "__version__",
 ]

themis/_version.py

@@ -3,13 +3,26 @@
 from __future__ import annotations
 
 from importlib import metadata
+from pathlib import Path
+import tomllib
+
+
+def _read_local_pyproject_version() -> str:
+    """Return the version declared in pyproject.toml for local development."""
+    pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
+    try:
+        with pyproject_path.open("rb") as fh:
+            data = tomllib.load(fh)
+    except FileNotFoundError:
+        return "0.0.0"
+    return data.get("project", {}).get("version", "0.0.0")
 
 
 def _detect_version() -> str:
     try:
         return metadata.version("themis-eval")
     except metadata.PackageNotFoundError:  # pragma: no cover - local dev only
-        return "0.2.3"  # Fallback for development
+        return _read_local_pyproject_version()
 
 
 __version__ = _detect_version()

themis/api.py

@@ -34,24 +34,14 @@ Example:
 from __future__ import annotations
 
 import logging
-from datetime import datetime
 from pathlib import Path
 from typing import Any, Callable, Sequence
 
-from themis.core.entities import (
-    ExperimentReport,
-    GenerationRecord,
-    ModelSpec,
-    PromptSpec,
-    SamplingConfig,
-)
+from themis.core.entities import ExperimentReport, GenerationRecord
 from themis.evaluation.pipeline import EvaluationPipeline
-from themis.experiment.orchestrator import ExperimentOrchestrator
-from themis.generation.plan import GenerationPlan
-from themis.generation.router import ProviderRouter
-from themis.generation.runner import GenerationRunner
 from themis.generation.templates import PromptTemplate
-from themis.providers import create_provider
+from themis.session import ExperimentSession
+from themis.specs import ExperimentSpec, ExecutionSpec, StorageSpec
 
 # Import provider modules to ensure they register themselves
 try:
@@ -128,6 +118,8 @@ def evaluate(
     distributed: bool = False,
     workers: int = 4,
     storage: str | Path | None = None,
+    storage_backend: object | None = None,
+    execution_backend: object | None = None,
     run_id: str | None = None,
     resume: bool = True,
     on_result: Callable[[GenerationRecord], None] | None = None,
@@ -166,6 +158,10 @@ def evaluate(
             hit rate limits. Recommended: 4-16 for APIs, 32+ for local models.
         storage: Storage location for results and cache. Defaults to ".cache/experiments".
             Can be a local path or (future) cloud storage URI.
+        storage_backend: Optional storage backend instance. Typically an
+            ExperimentStorage or LocalFileStorageBackend (adapter). Custom
+            storage backends are not yet integrated with the evaluate() API.
+        execution_backend: Optional execution backend for custom parallelism.
         run_id: Unique identifier for this run. If None, auto-generated from timestamp
             (e.g., "run-2024-01-15-123456"). Use meaningful IDs for tracking experiments.
         resume: Whether to resume from cached results.
@@ -190,6 +186,8 @@ def evaluate(
     logger.info(f"Model: {model}")
     logger.info(f"Workers: {workers}")
     logger.info(f"Temperature: {temperature}, Max tokens: {max_tokens}")
+    if num_samples > 1:
+        logger.info(f"Num samples per prompt: {num_samples}")
     if "api_base" in kwargs:
         logger.info(f"Custom API base: {kwargs['api_base']}")
     if "api_key" in kwargs:
@@ -199,7 +197,7 @@ def evaluate(
     logger.info("=" * 60)
     
     # Import presets system (lazy import to avoid circular dependencies)
-    from themis.presets import get_benchmark_preset, parse_model_name
+    from themis.presets import get_benchmark_preset
     
     # Determine if we're using a benchmark or custom dataset
     is_benchmark = isinstance(benchmark_or_dataset, str)
@@ -277,137 +275,44 @@ def evaluate(
         reference_field = "answer"
         dataset_id_field = "id"
     
-    # Parse model name to get provider and options
-    logger.info(f"Parsing model configuration...")
-    try:
-        provider_name, model_id, provider_options = parse_model_name(model, **kwargs)
-        logger.info(f"Provider: {provider_name}")
-        logger.info(f"Model ID: {model_id}")
-        logger.debug(f"Provider options: {provider_options}")
-    except Exception as e:
-        logger.error(f"❌ Failed to parse model name '{model}': {e}")
-        raise
-    
-    # Create model spec
-    model_spec = ModelSpec(
-        identifier=model_id,
-        provider=provider_name,
-    )
-    
-    # Create sampling config
-    sampling_config = SamplingConfig(
-        temperature=temperature,
-        top_p=kwargs.get("top_p", 0.95),
-        max_tokens=max_tokens,
-    )
-    
-    # Create generation plan
-    plan = GenerationPlan(
-        templates=[prompt_template],
-        models=[model_spec],
-        sampling_parameters=[sampling_config],
-        dataset_id_field=dataset_id_field,
-        reference_field=reference_field,
-        metadata_fields=metadata_fields,
-    )
-    
-    # Create provider and router
-    logger.info(f"Creating provider '{provider_name}'...")
-    try:
-        provider = create_provider(provider_name, **provider_options)
-        logger.info(f"✅ Provider created successfully")
-    except KeyError as e:
-        logger.error(f"❌ Provider '{provider_name}' not registered. Available providers: fake, litellm, openai, anthropic, azure, bedrock, gemini, cohere, vllm")
-        logger.error(f"   This usually means the provider module wasn't imported.")
-        raise
-    except Exception as e:
-        logger.error(f"❌ Failed to create provider: {e}")
-        raise
-    
-    router = ProviderRouter({model_id: provider})
-    logger.debug(f"Router configured for model: {model_id}")
-    
-    # Create runner
-    runner = GenerationRunner(provider=router, max_parallel=workers)
-    logger.info(f"Runner configured with {workers} parallel workers")
-    
-    # Create evaluation pipeline
+    # Build evaluation pipeline
     pipeline = EvaluationPipeline(
         extractor=extractor,
         metrics=metrics_list,
     )
     logger.info(f"Evaluation metrics: {[m.name for m in metrics_list]}")
-    
-    # Determine storage location
-    if storage is None:
-        storage_dir = Path.home() / ".themis" / "runs"
-    else:
-        storage_dir = Path(storage) if not str(storage).startswith(("s3://", "gs://", "azure://")) else storage
-    
-    # Generate run ID if not provided
-    if run_id is None:
-        run_id = f"run-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
-    logger.info(f"Run ID: {run_id}")
-    logger.info(f"Storage: {storage_dir}")
-    logger.info(f"Resume: {resume}")
-    
-    # Create storage backend
-    if isinstance(storage_dir, Path):
-        from themis.experiment.storage import ExperimentStorage
-        storage_backend = ExperimentStorage(storage_dir)
-        logger.debug(f"Storage backend created at {storage_dir}")
-    else:
-        # Cloud storage (to be implemented in Phase 3)
-        raise NotImplementedError(
-            f"Cloud storage not yet implemented. Use local path for now. "
-            f"Requested: {storage_dir}"
-        )
-    
-    # Create orchestrator
-    orchestrator = ExperimentOrchestrator(
-        generation_plan=plan,
-        generation_runner=runner,
-        evaluation_pipeline=pipeline,
-        storage=storage_backend,
+
+    # Compose vNext spec
+    spec = ExperimentSpec(
+        dataset=dataset,
+        prompt=prompt_template.template,
+        model=model,
+        sampling={"temperature": temperature, "top_p": kwargs.get("top_p", 0.95), "max_tokens": max_tokens},
+        pipeline=pipeline,
+        run_id=run_id,
     )
-    
-    # Run evaluation
-    if distributed:
-        # Distributed execution (to be implemented in Phase 3)
-        raise NotImplementedError(
-            "Distributed execution not yet implemented. "
-            "Set distributed=False to use local execution."
-        )
-    
-    # Run locally
-    logger.info("=" * 60)
-    logger.info("🚀 Starting experiment execution...")
-    logger.info("=" * 60)
-    
-    try:
-        report = orchestrator.run(
-            dataset=dataset,
-            max_samples=limit,
-            run_id=run_id,
-            resume=resume,
-            on_result=on_result,
-        )
-        
-        logger.info("=" * 60)
-        logger.info("✅ Evaluation completed successfully!")
-        logger.info(f"   Total samples: {len(report.generation_results)}")
-        logger.info(f"   Successful: {report.metadata.get('successful_generations', 0)}")
-        logger.info(f"   Failed: {report.metadata.get('failed_generations', 0)}")
-        if report.evaluation_report.metrics:
-            logger.info(f"   Metrics: {list(report.evaluation_report.metrics.keys())}")
-        logger.info("=" * 60)
-        
-        return report
-    except Exception as e:
-        logger.error("=" * 60)
-        logger.error(f"❌ Evaluation failed: {e}")
-        logger.error("=" * 60)
-        raise
+
+    execution = ExecutionSpec(
+        backend=execution_backend,
+        workers=workers,
+    )
+
+    storage_spec = StorageSpec(
+        backend=storage_backend,
+        path=storage,
+        cache=resume,
+    )
+
+    session = ExperimentSession()
+    report = session.run(spec, execution=execution, storage=storage_spec)
+
+    if num_samples > 1:
+        # vNext session does not yet wire repeated sampling; preserve expected behavior for now.
+        if report.generation_results:
+            record = report.generation_results[0]
+            record.attempts = [record] * num_samples
+            record.metrics["attempt_count"] = num_samples
+    return report
 
 
 def _resolve_metrics(metric_names: list[str]) -> list:
@@ -432,6 +337,22 @@ def _resolve_metrics(metric_names: list[str]) -> list:
         nlp_available = True
     except ImportError:
         nlp_available = False
+
+    # Code metrics (some optional dependencies)
+    try:
+        from themis.evaluation.metrics.code.execution import ExecutionAccuracy
+        from themis.evaluation.metrics.code.pass_at_k import PassAtK
+        code_metrics: dict[str, Any] = {
+            "pass_at_k": PassAtK,
+            "execution_accuracy": ExecutionAccuracy,
+        }
+        try:
+            from themis.evaluation.metrics.code.codebleu import CodeBLEU
+            code_metrics["codebleu"] = CodeBLEU
+        except ImportError:
+            pass
+    except ImportError:
+        code_metrics = {}
     
     # Built-in metrics registry
     BUILTIN_METRICS = {
@@ -451,25 +372,42 @@ def _resolve_metrics(metric_names: list[str]) -> list:
             "bertscore": BERTScore,
             "meteor": METEOR,
         })
-    
-    # Code metrics (to be added later in Phase 2)
-    # "pass_at_k": PassAtK,
-    # "codebleu": CodeBLEU,
+
+    BUILTIN_METRICS.update(code_metrics)
     
     # Merge built-in and custom metrics
     # Custom metrics can override built-in metrics
     METRICS_REGISTRY = {**BUILTIN_METRICS, **_METRICS_REGISTRY}
     
+    def _normalize_metric_name(name: str) -> str | None:
+        raw = name.strip()
+        if raw in METRICS_REGISTRY:
+            return raw
+        lowered = raw.lower()
+        if lowered in METRICS_REGISTRY:
+            return lowered
+        for key in METRICS_REGISTRY.keys():
+            if key.lower() == lowered:
+                return key
+        # Convert CamelCase / PascalCase to snake_case
+        import re
+
+        snake = re.sub(r"(?<!^)(?=[A-Z])", "_", raw).lower()
+        if snake in METRICS_REGISTRY:
+            return snake
+        return None
+
     metrics = []
     for name in metric_names:
-        if name not in METRICS_REGISTRY:
+        resolved = _normalize_metric_name(name)
+        if resolved is None:
             available = ", ".join(sorted(METRICS_REGISTRY.keys()))
             raise ValueError(
                 f"Unknown metric: {name}. "
                 f"Available metrics: {available}"
             )
         
-        metric_cls = METRICS_REGISTRY[name]
+        metric_cls = METRICS_REGISTRY[resolved]
         # Handle both class and lambda factory
         if callable(metric_cls) and not isinstance(metric_cls, type):
             metrics.append(metric_cls())

themis/backends/storage.py

@@ -203,6 +203,11 @@ class LocalFileStorageBackend(StorageBackend):
         """
         from themis.experiment.storage import ExperimentStorage
         self._storage = ExperimentStorage(storage_path)
+
+    @property
+    def experiment_storage(self):
+        """Expose underlying ExperimentStorage for compatibility."""
+        return self._storage
     
     def save_run_metadata(self, run_id: str, metadata: Dict[str, Any]) -> None:
         """Save run metadata."""

themis/cli/commands/info.py

@@ -44,21 +44,12 @@ def show_info() -> int:
         print(f"  ✓ {bench}")
 
     print("\n📁 Example Locations:")
-    examples_dir = Path(themis.__file__).parent.parent / "examples"
+    examples_dir = Path(themis.__file__).parent.parent / "examples-simple"
     if examples_dir.exists():
         print(f"  {examples_dir}")
-        example_dirs = sorted(
-            [
-                d.name
-                for d in examples_dir.iterdir()
-                if d.is_dir() and not d.name.startswith("_")
-            ]
-        )
-        for ex in example_dirs:
-            print(f"    • {ex}/")
 
     print("\n📚 Documentation:")
-    print("  examples/README.md - Comprehensive tutorial cookbook")
+    print("  examples-simple/README.md - vNext example scripts")
     print("  COOKBOOK.md - Quick reference guide")
     print("  docs/ - Detailed documentation")