themis-eval 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

themis/experiment/export_csv.py

@@ -0,0 +1,159 @@
+"""CSV export functionality for experiment reports."""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import MutableMapping, Sequence
+
+from themis.core import entities as core_entities
+from themis.experiment import orchestrator
+
+
+def export_report_csv(
+    report: orchestrator.ExperimentReport,
+    path: str | Path,
+    *,
+    include_failures: bool = True,
+) -> Path:
+    """Write per-sample metrics to a CSV file for offline analysis.
+
+    Args:
+        report: Experiment report to export
+        path: Output path for CSV file
+        include_failures: Whether to include failures column
+
+    Returns:
+        Path to created CSV file
+    """
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    metadata_by_condition, metadata_fields = _collect_sample_metadata(
+        report.generation_results
+    )
+
+    # Create a proper index mapping generation records to their metadata
+    gen_record_index = {}
+    for gen_record in report.generation_results:
+        sample_id = gen_record.task.metadata.get(
+            "dataset_id"
+        ) or gen_record.task.metadata.get("sample_id")
+        prompt_template = gen_record.task.prompt.spec.name
+        model_identifier = gen_record.task.model.identifier
+        sampling_temp = gen_record.task.sampling.temperature
+        sampling_max_tokens = gen_record.task.sampling.max_tokens
+        condition_id = f"{sample_id}_{prompt_template}_{model_identifier}_{sampling_temp}_{sampling_max_tokens}"
+        gen_record_index[condition_id] = gen_record
+
+    metric_names = sorted(report.evaluation_report.metrics.keys())
+    fieldnames = (
+        ["sample_id"] + metadata_fields + [f"metric:{name}" for name in metric_names]
+    )
+    if include_failures:
+        fieldnames.append("failures")
+
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+
+        # Process evaluation records in the same order as generation records
+        for i, eval_record in enumerate(report.evaluation_report.records):
+            # Find the corresponding generation record by index
+            if i < len(report.generation_results):
+                gen_record = report.generation_results[i]
+                sample_id = gen_record.task.metadata.get(
+                    "dataset_id"
+                ) or gen_record.task.metadata.get("sample_id")
+                prompt_template = gen_record.task.prompt.spec.name
+                model_identifier = gen_record.task.model.identifier
+                sampling_temp = gen_record.task.sampling.temperature
+                sampling_max_tokens = gen_record.task.sampling.max_tokens
+                condition_id = f"{sample_id}_{prompt_template}_{model_identifier}_{sampling_temp}_{sampling_max_tokens}"
+                metadata = metadata_by_condition.get(condition_id, {})
+            else:
+                # Fallback for extra evaluation records
+                sample_id = eval_record.sample_id or ""
+                metadata = {}
+
+            row: dict[str, object] = {"sample_id": sample_id}
+            for field in metadata_fields:
+                row[field] = metadata.get(field, "")
+            score_by_name = {
+                score.metric_name: score.value for score in eval_record.scores
+            }
+            for name in metric_names:
+                row[f"metric:{name}"] = score_by_name.get(name, "")
+            if include_failures:
+                row["failures"] = "; ".join(eval_record.failures)
+            writer.writerow(row)
+    return path
+
+
+def _collect_sample_metadata(
+    records: Sequence[core_entities.GenerationRecord],
+) -> tuple[dict[str, MutableMapping[str, object]], list[str]]:
+    """Collect metadata from generation records.
+
+    Args:
+        records: Generation records
+
+    Returns:
+        Tuple of (metadata by condition ID, list of metadata fields)
+    """
+    metadata: dict[str, MutableMapping[str, object]] = {}
+    for index, record in enumerate(records):
+        sample_id = _extract_sample_id(record.task.metadata)
+        if sample_id is None:
+            sample_id = f"sample-{index}"
+
+        # Create unique identifier for each experimental condition
+        prompt_template = record.task.prompt.spec.name
+        model_identifier = record.task.model.identifier
+        sampling_temp = record.task.sampling.temperature
+        sampling_max_tokens = record.task.sampling.max_tokens
+
+        # Create unique condition key
+        condition_id = f"{sample_id}_{prompt_template}_{model_identifier}_{sampling_temp}_{sampling_max_tokens}"
+
+        # Store metadata with unique condition ID
+        condition_metadata = _metadata_from_task(record)
+        metadata[condition_id] = condition_metadata
+
+    # Collect all field names from all conditions
+    fields = sorted({field for meta in metadata.values() for field in meta.keys()})
+
+    return metadata, fields
+
+
+def _extract_sample_id(metadata: dict[str, object]) -> str | None:
+    """Extract sample ID from metadata.
+
+    Args:
+        metadata: Task metadata
+
+    Returns:
+        Sample ID or None
+    """
+    value = metadata.get("dataset_id") or metadata.get("sample_id")
+    if value is None:
+        return None
+    return str(value)
+
+
+def _metadata_from_task(record: core_entities.GenerationRecord) -> dict[str, object]:
+    """Build metadata dict from generation record.
+
+    Args:
+        record: Generation record
+
+    Returns:
+        Metadata dictionary
+    """
+    metadata = dict(record.task.metadata)
+    metadata.setdefault("model_identifier", record.task.model.identifier)
+    metadata.setdefault("model_provider", record.task.model.provider)
+    metadata.setdefault("prompt_template", record.task.prompt.spec.name)
+    metadata.setdefault("sampling_temperature", record.task.sampling.temperature)
+    metadata.setdefault("sampling_top_p", record.task.sampling.top_p)
+    metadata.setdefault("sampling_max_tokens", record.task.sampling.max_tokens)
+    return metadata

themis/experiment/integration_manager.py

@@ -0,0 +1,104 @@
+"""Integration management for external services (WandB, HuggingFace Hub)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from themis.config.schema import IntegrationsConfig
+from themis.core.entities import ExperimentReport
+from themis.integrations.huggingface import HuggingFaceHubUploader
+from themis.integrations.wandb import WandbTracker
+
+
+class IntegrationManager:
+    """Manages external integrations (WandB, HuggingFace Hub).
+
+    This class handles all integration-related operations including:
+    - Initializing integrations based on configuration
+    - Logging experiment results to WandB
+    - Uploading results to HuggingFace Hub
+    - Finalizing integrations on completion
+
+    Single Responsibility: External integration management
+    """
+
+    def __init__(self, config: IntegrationsConfig | None = None) -> None:
+        """Initialize integration manager.
+
+        Args:
+            config: Integration configuration (None disables all integrations)
+        """
+        self._config = config or IntegrationsConfig()
+
+        # Initialize WandB tracker if enabled
+        self._wandb_tracker = (
+            WandbTracker(self._config.wandb) if self._config.wandb.enable else None
+        )
+
+        # Initialize HuggingFace Hub uploader if enabled
+        self._hf_uploader = (
+            HuggingFaceHubUploader(self._config.huggingface_hub)
+            if self._config.huggingface_hub.enable
+            else None
+        )
+
+    @property
+    def has_wandb(self) -> bool:
+        """Check if WandB integration is enabled."""
+        return self._wandb_tracker is not None
+
+    @property
+    def has_huggingface(self) -> bool:
+        """Check if HuggingFace Hub integration is enabled."""
+        return self._hf_uploader is not None
+
+    def initialize_run(self, run_config: dict[str, Any]) -> None:
+        """Initialize integrations for a new run.
+
+        Args:
+            run_config: Configuration dictionary for the run
+                Common keys: max_samples, run_id, resume
+        """
+        if self._wandb_tracker:
+            self._wandb_tracker.init(run_config)
+
+    def log_results(self, report: ExperimentReport) -> None:
+        """Log experiment results to integrations.
+
+        Args:
+            report: Completed experiment report with all results
+        """
+        if self._wandb_tracker:
+            self._wandb_tracker.log_results(report)
+
+    def upload_results(
+        self,
+        report: ExperimentReport,
+        run_path: str | Path | None,
+    ) -> None:
+        """Upload results to HuggingFace Hub.
+
+        Args:
+            report: Completed experiment report
+            run_path: Path to run directory with cached results
+        """
+        if self._hf_uploader and run_path is not None:
+            self._hf_uploader.upload_results(report, run_path)
+
+    def finalize(self) -> None:
+        """Finalize all integrations.
+
+        This should be called after experiment completion to properly
+        close connections and clean up resources.
+        """
+        if self._wandb_tracker:
+            # WandB tracker handles finalization in log_results
+            pass
+
+        if self._hf_uploader:
+            # HuggingFace uploader is stateless, no finalization needed
+            pass
+
+
+__all__ = ["IntegrationManager"]

themis/experiment/math.py

@@ -0,0 +1,192 @@
+"""High-level helpers for math-focused experiments."""
+
+from __future__ import annotations
+
+from textwrap import dedent
+from typing import Sequence
+
+from themis.core import entities as core_entities
+from themis.evaluation import extractors, math_verify_utils, metrics, pipeline
+from themis.experiment import orchestrator
+from themis.experiment import storage as experiment_storage
+from themis.generation import clients, plan, runner, templates
+from themis.interfaces import ModelProvider
+
+
+def build_math500_zero_shot_experiment(
+    *,
+    model_client: ModelProvider | None = None,
+    model_name: str = "fake-math-llm",
+    provider_name: str = "fake",
+    temperature: float | None = None,
+    sampling: core_entities.SamplingConfig | None = None,
+    storage: experiment_storage.ExperimentStorage | None = None,
+    runner_options: dict[str, object] | None = None,
+    task_name: str = "math500",
+) -> orchestrator.ExperimentOrchestrator:
+    """Create an experiment orchestrator tailored for competition math benchmarks."""
+
+    prompt_template = templates.PromptTemplate(
+        name=f"{task_name}-zero-shot-json",
+        template=dedent(
+            """
+            You are an expert competition mathematician. Solve the following problem in a zero-shot
+            manner. Think carefully and provide a short reasoning paragraph followed by a line of the
+            form `Final Answer: \\boxed{{value}}` where `value` is the final numeric result.
+
+            Problem:
+            {problem}
+            """
+        ).strip(),
+        metadata={"task": task_name, "expect_boxed": True},
+    )
+
+    sampling = sampling or core_entities.SamplingConfig(
+        temperature=temperature if temperature is not None else 0.0,
+        top_p=0.95,
+        max_tokens=512,
+    )
+    model_spec = core_entities.ModelSpec(
+        identifier=model_name, provider=provider_name, default_sampling=sampling
+    )
+    math_plan = plan.GenerationPlan(
+        templates=[prompt_template],
+        models=[model_spec],
+        sampling_parameters=[sampling],
+        dataset_id_field="unique_id",
+        reference_field="answer",
+        metadata_fields=("subject", "level"),
+        context_builder=lambda row: {"problem": row.get("problem", "")},
+    )
+
+    # Extract runner options with proper type conversion
+    runner_kwargs = {}
+    if runner_options:
+        # Convert values to appropriate types with type checking
+        if (
+            "max_parallel" in runner_options
+            and runner_options["max_parallel"] is not None
+        ):
+            runner_kwargs["max_parallel"] = int(str(runner_options["max_parallel"]))
+        if (
+            "max_retries" in runner_options
+            and runner_options["max_retries"] is not None
+        ):
+            runner_kwargs["max_retries"] = int(str(runner_options["max_retries"]))
+        if (
+            "retry_initial_delay" in runner_options
+            and runner_options["retry_initial_delay"] is not None
+        ):
+            runner_kwargs["retry_initial_delay"] = float(
+                str(runner_options["retry_initial_delay"])
+            )
+        if (
+            "retry_backoff_multiplier" in runner_options
+            and runner_options["retry_backoff_multiplier"] is not None
+        ):
+            runner_kwargs["retry_backoff_multiplier"] = float(
+                str(runner_options["retry_backoff_multiplier"])
+            )
+        if "retry_max_delay" in runner_options:
+            retry_max_delay = runner_options["retry_max_delay"]
+            runner_kwargs["retry_max_delay"] = (
+                float(str(retry_max_delay)) if retry_max_delay is not None else None
+            )
+
+    math_runner = runner.GenerationRunner(
+        provider=model_client or clients.FakeMathModelClient(),
+        **runner_kwargs,
+    )
+    if math_verify_utils.math_verify_available():
+        extractor = extractors.MathVerifyExtractor()
+        metric_list = [
+            metrics.MathVerifyAccuracy(),
+            metrics.ExactMatch(case_sensitive=False, strip_whitespace=True),
+        ]
+    else:
+        extractor = extractors.JsonFieldExtractor(field_path="answer")
+        metric_list = [
+            metrics.ExactMatch(case_sensitive=False, strip_whitespace=True),
+        ]
+    eval_pipeline = pipeline.EvaluationPipeline(
+        extractor=extractor,
+        metrics=metric_list,
+    )
+
+    return orchestrator.ExperimentOrchestrator(
+        generation_plan=math_plan,
+        generation_runner=math_runner,
+        evaluation_pipeline=eval_pipeline,
+        storage=storage,
+    )
+
+
+def run_math500_zero_shot(
+    dataset: Sequence[dict[str, object]],
+    *,
+    model_client: clients.FakeMathModelClient | None = None,
+    max_samples: int | None = None,
+    storage: experiment_storage.ExperimentStorage | None = None,
+    run_id: str | None = None,
+    resume: bool = True,
+) -> orchestrator.ExperimentReport:
+    """Run the zero-shot math experiment against a prepared dataset."""
+
+    experiment = build_math500_zero_shot_experiment(
+        model_client=model_client, storage=storage
+    )
+    return experiment.run(
+        dataset, max_samples=max_samples, run_id=run_id, resume=resume
+    )
+
+
+def summarize_report(report: orchestrator.ExperimentReport) -> str:
+    # Get exact match metric
+    exact = report.evaluation_report.metrics.get("ExactMatch")
+    exact_mean = exact.mean if exact else 0.0
+    exact_count = exact.count if exact else 0
+
+    # Get MathVerify metric if available
+    math_verify = report.evaluation_report.metrics.get("MathVerifyAccuracy")
+    math_verify_mean = math_verify.mean if math_verify else None
+    math_verify_count = math_verify.count if math_verify else 0
+
+    # Get failure counts
+    generation_failures = len(report.failures)
+    evaluation_failures = len(report.evaluation_report.failures)
+    total_failures = generation_failures + evaluation_failures
+
+    # Get metadata
+    total_samples = report.metadata.get("total_samples", 0)
+    successful_generations = report.metadata.get("successful_generations", 0)
+    failed_generations = report.metadata.get("failed_generations", 0)
+
+    # Build summary string
+    summary_parts = [
+        f"Evaluated {total_samples} samples",
+        f"Successful generations: {successful_generations}/{total_samples}",
+        f"Exact match: {exact_mean:.3f} ({exact_count} evaluated)",
+    ]
+
+    # Add MathVerify accuracy if available
+    if math_verify_mean is not None:
+        summary_parts.append(
+            f"MathVerify accuracy: {math_verify_mean:.3f} ({math_verify_count} evaluated)"
+        )
+
+    # Add failure information
+    if total_failures > 0:
+        summary_parts.append(
+            f"Failures: {total_failures} (gen: {failed_generations}, eval: {evaluation_failures})"
+        )
+    else:
+        summary_parts.append("No failures")
+
+    return " | ".join(summary_parts)
+
+
+__all__ = [
+    "build_math500_zero_shot_experiment",
+    "run_math500_zero_shot",
+    "summarize_report",
+]

themis/experiment/mcq.py

@@ -0,0 +1,169 @@
+"""Experiment builders for multiple-choice benchmarks."""
+
+from __future__ import annotations
+
+from textwrap import dedent
+from typing import Callable, Sequence
+
+from themis.core import entities as core_entities
+from themis.evaluation import extractors, metrics, pipeline
+from themis.experiment import orchestrator
+from themis.experiment import storage as experiment_storage
+from themis.generation import clients, plan, runner, templates
+from themis.interfaces import ModelProvider
+
+
+def build_multiple_choice_json_experiment(
+    *,
+    dataset_name: str,
+    task_id: str | None = None,
+    model_client: ModelProvider | None = None,
+    model_name: str = "fake-math-llm",
+    provider_name: str = "fake",
+    temperature: float | None = None,
+    sampling: core_entities.SamplingConfig | None = None,
+    storage: experiment_storage.ExperimentStorage | None = None,
+    runner_options: dict[str, object] | None = None,
+    metadata_fields: Sequence[str] = ("subject",),
+    context_builder: Callable[[dict[str, object]], dict[str, object]] | None = None,
+) -> orchestrator.ExperimentOrchestrator:
+    """Create an experiment orchestrator for multiple-choice QA benchmarks."""
+
+    task_id = task_id or dataset_name
+    prompt_template = templates.PromptTemplate(
+        name=f"{dataset_name}-multiple-choice-json",
+        template=dedent(
+            """
+            You are an expert test taker. Select the single best answer to the following
+            multiple-choice question.
+
+            Question:
+            {question}
+
+            Choices:
+            {choices_block}
+
+            Respond with a JSON object containing two keys:
+              "answer" - the capital letter of the chosen option (e.g. "A")
+              "explanation" - one or two sentences explaining your reasoning
+
+            Example response:
+            {{"answer": "A", "explanation": "Reasoning..."}}
+            """
+        ).strip(),
+        metadata={"task": task_id, "response_format": "json"},
+    )
+
+    sampling = sampling or core_entities.SamplingConfig(
+        temperature=temperature if temperature is not None else 0.0,
+        top_p=0.95,
+        max_tokens=512,
+    )
+    model_spec = core_entities.ModelSpec(
+        identifier=model_name, provider=provider_name, default_sampling=sampling
+    )
+
+    def _default_context_builder(row: dict[str, object]) -> dict[str, object]:
+        labels: Sequence[str] = tuple(
+            str(label) for label in row.get("choice_labels", [])
+        ) or tuple("ABCD")
+        choices: Sequence[str] = tuple(str(choice) for choice in row.get("choices", []))
+        choice_lines = []
+        for label, choice in zip(labels, choices, strict=False):
+            choice_lines.append(f"{label}. {choice}")
+        choices_block = "\n".join(choice_lines)
+        return {
+            "question": str(row.get("question", "")),
+            "choices_block": choices_block,
+        }
+
+    mcq_plan = plan.GenerationPlan(
+        templates=[prompt_template],
+        models=[model_spec],
+        sampling_parameters=[sampling],
+        dataset_id_field="unique_id",
+        reference_field="answer",
+        metadata_fields=tuple(metadata_fields),
+        context_builder=context_builder or _default_context_builder,
+    )
+
+    runner_kwargs = {}
+    if runner_options:
+        if (
+            "max_parallel" in runner_options
+            and runner_options["max_parallel"] is not None
+        ):
+            runner_kwargs["max_parallel"] = int(str(runner_options["max_parallel"]))
+        if (
+            "max_retries" in runner_options
+            and runner_options["max_retries"] is not None
+        ):
+            runner_kwargs["max_retries"] = int(str(runner_options["max_retries"]))
+        if (
+            "retry_initial_delay" in runner_options
+            and runner_options["retry_initial_delay"] is not None
+        ):
+            runner_kwargs["retry_initial_delay"] = float(
+                str(runner_options["retry_initial_delay"])
+            )
+        if (
+            "retry_backoff_multiplier" in runner_options
+            and runner_options["retry_backoff_multiplier"] is not None
+        ):
+            runner_kwargs["retry_backoff_multiplier"] = float(
+                str(runner_options["retry_backoff_multiplier"])
+            )
+        if "retry_max_delay" in runner_options:
+            retry_max_delay = runner_options["retry_max_delay"]
+            runner_kwargs["retry_max_delay"] = (
+                float(str(retry_max_delay)) if retry_max_delay is not None else None
+            )
+
+    mcq_runner = runner.GenerationRunner(
+        provider=model_client or clients.FakeMathModelClient(),
+        **runner_kwargs,
+    )
+
+    extractor = extractors.JsonFieldExtractor(field_path="answer")
+    metric_list = [
+        metrics.ExactMatch(case_sensitive=False, strip_whitespace=True),
+    ]
+    eval_pipeline = pipeline.EvaluationPipeline(
+        extractor=extractor,
+        metrics=metric_list,
+    )
+
+    return orchestrator.ExperimentOrchestrator(
+        generation_plan=mcq_plan,
+        generation_runner=mcq_runner,
+        evaluation_pipeline=eval_pipeline,
+        storage=storage,
+    )
+
+
+def summarize_report(report: orchestrator.ExperimentReport) -> str:
+    exact = report.evaluation_report.metrics.get("ExactMatch")
+    accuracy = exact.mean if exact else 0.0
+    evaluated = exact.count if exact else 0
+
+    total_samples = report.metadata.get("total_samples", evaluated)
+    successful_generations = report.metadata.get("successful_generations", evaluated)
+    failed_generations = report.metadata.get("failed_generations", 0)
+    evaluation_failures = len(report.evaluation_report.failures)
+    total_failures = failed_generations + evaluation_failures
+
+    summary_parts = [
+        f"Evaluated {total_samples} samples",
+        f"Successful generations: {successful_generations}/{total_samples}",
+        f"Accuracy: {accuracy:.3f} ({evaluated} evaluated)",
+    ]
+    if total_failures:
+        summary_parts.append(
+            f"Failures: {total_failures} (gen: {failed_generations}, eval: {evaluation_failures})"
+        )
+    else:
+        summary_parts.append("No failures")
+    return " | ".join(summary_parts)
+
+
+__all__ = ["build_multiple_choice_json_experiment", "summarize_report"]