themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/config/schema.py

@@ -0,0 +1,112 @@
+"""Structured configuration definitions for Hydra/OmegaConf."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class ProviderConfig:
+    name: str = "fake"
+    options: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RunnerConfig:
+    max_parallel: int = 1
+    max_retries: int = 3
+    retry_initial_delay: float = 0.5
+    retry_backoff_multiplier: float = 2.0
+    retry_max_delay: float | None = 2.0
+
+
+@dataclass
+class SamplingConfig:
+    temperature: float = 0.0
+    top_p: float = 0.95
+    max_tokens: int = 512
+
+
+@dataclass
+class GenerationConfig:
+    model_identifier: str = "fake-math-llm"
+    provider: ProviderConfig = field(default_factory=ProviderConfig)
+    sampling: SamplingConfig = field(default_factory=SamplingConfig)
+    runner: RunnerConfig = field(default_factory=RunnerConfig)
+
+
+@dataclass
+class DatasetConfig:
+    source: str = "huggingface"
+    dataset_id: str | None = None
+    data_dir: str | None = None
+    limit: int | None = None
+    split: str = "test"
+    subjects: list[str] = field(default_factory=list)
+    inline_samples: list[dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass
+class StorageConfig:
+    path: str | None = None
+    default_path: str | None = None  # New field for default storage path
+
+
+@dataclass
+class WandbConfig:
+    enable: bool = False
+    project: str | None = None
+    entity: str | None = None
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class HuggingFaceHubConfig:
+    enable: bool = False
+    repository: str | None = None
+
+
+@dataclass
+class IntegrationsConfig:
+    wandb: WandbConfig = field(default_factory=WandbConfig)
+    huggingface_hub: HuggingFaceHubConfig = field(default_factory=HuggingFaceHubConfig)
+
+
+@dataclass
+class ExperimentConfig:
+    name: str = "math500_zero_shot"
+    dataset: DatasetConfig = field(default_factory=DatasetConfig)
+    generation: GenerationConfig = field(default_factory=GenerationConfig)
+    storage: StorageConfig = field(default_factory=StorageConfig)
+    integrations: IntegrationsConfig = field(default_factory=IntegrationsConfig)
+    max_samples: int | None = None
+    run_id: str | None = None
+    resume: bool = True
+    task: str | None = None
+    task_options: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> ExperimentConfig:
+        """Load configuration from a file."""
+        from .loader import load_experiment_config
+
+        return load_experiment_config(Path(path))
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ExperimentConfig:
+        """Create configuration from a dictionary."""
+        from omegaconf import OmegaConf
+
+        base = OmegaConf.structured(cls)
+        merged = OmegaConf.merge(base, OmegaConf.create(data))
+        return OmegaConf.to_object(merged)  # type: ignore
+
+    def to_file(self, path: str | Path) -> None:
+        """Save configuration to a file."""
+        from omegaconf import OmegaConf
+
+        conf = OmegaConf.structured(self)
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        OmegaConf.save(conf, Path(path))

themis/core/__init__.py

@@ -0,0 +1,5 @@
+"""Core datamodel for Themis."""
+
+from . import entities, serialization
+
+__all__ = ["entities", "serialization"]

themis/core/conversation.py

@@ -0,0 +1,354 @@
+"""Conversation primitives for multi-turn interactions.
+
+This module provides abstractions for managing multi-turn conversations,
+enabling research on dialogue systems, debugging interactions, and
+agentic workflows.
+
+Examples:
+    # Create a conversation
+    context = ConversationContext()
+    context.add_message("user", "What is 2+2?")
+    context.add_message("assistant", "2+2 equals 4.")
+    context.add_message("user", "What about 3+3?")
+
+    # Convert to prompt
+    prompt = context.to_prompt()
+
+    # Get conversation history
+    history = context.get_history(max_turns=2)
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Literal
+
+from themis.core import entities as core_entities
+from themis.generation import templates
+
+MessageRole = Literal["system", "user", "assistant", "tool"]
+
+
+@dataclass
+class Message:
+    """Single message in a conversation.
+
+    Attributes:
+        role: Message role (system/user/assistant/tool)
+        content: Message text content
+        metadata: Additional metadata (tool calls, timestamps, etc.)
+        timestamp: Unix timestamp when message was created
+    """
+
+    role: MessageRole
+    content: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    timestamp: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert message to dictionary."""
+        return {
+            "role": self.role,
+            "content": self.content,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp,
+        }
+
+
+@dataclass
+class ConversationContext:
+    """Maintains conversation state across turns.
+
+    This class manages the conversation history and provides utilities
+    for rendering conversations as prompts.
+
+    Examples:
+        context = ConversationContext()
+        context.add_message("system", "You are a helpful assistant.")
+        context.add_message("user", "Hello!")
+        context.add_message("assistant", "Hi! How can I help you?")
+
+        # Get history
+        messages = context.get_history()
+
+        # Render to prompt
+        prompt = context.to_prompt()
+    """
+
+    messages: list[Message] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def add_message(self, role: MessageRole, content: str, **metadata: Any) -> None:
+        """Add a message to the conversation.
+
+        Args:
+            role: Message role (system/user/assistant/tool)
+            content: Message text content
+            **metadata: Additional metadata to attach to message
+        """
+        self.messages.append(Message(role=role, content=content, metadata=metadata))
+
+    def get_history(self, max_turns: int | None = None) -> list[Message]:
+        """Get conversation history.
+
+        Args:
+            max_turns: Maximum number of messages to return (from end)
+
+        Returns:
+            List of messages (most recent if limited)
+        """
+        if max_turns is None:
+            return list(self.messages)
+        return self.messages[-max_turns:]
+
+    def get_messages_by_role(self, role: MessageRole) -> list[Message]:
+        """Get all messages with a specific role.
+
+        Args:
+            role: Role to filter by
+
+        Returns:
+            List of messages with matching role
+        """
+        return [msg for msg in self.messages if msg.role == role]
+
+    def to_prompt(self, template: templates.PromptTemplate | None = None) -> str:
+        """Render conversation to prompt string.
+
+        Args:
+            template: Optional template for custom formatting
+
+        Returns:
+            Formatted prompt string
+        """
+        if template is not None:
+            return template.render(messages=self.messages)
+
+        # Default format: role-prefixed messages
+        lines = []
+        for msg in self.messages:
+            lines.append(f"{msg.role}: {msg.content}")
+
+        return "\n\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert conversation to dictionary.
+
+        Returns:
+            Dictionary representation
+        """
+        return {
+            "messages": [msg.to_dict() for msg in self.messages],
+            "metadata": self.metadata,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ConversationContext:
+        """Create conversation from dictionary.
+
+        Args:
+            data: Dictionary with messages and metadata
+
+        Returns:
+            ConversationContext instance
+        """
+        context = cls(metadata=data.get("metadata", {}))
+        for msg_data in data.get("messages", []):
+            context.messages.append(
+                Message(
+                    role=msg_data["role"],
+                    content=msg_data["content"],
+                    metadata=msg_data.get("metadata", {}),
+                    timestamp=msg_data.get("timestamp", time.time()),
+                )
+            )
+        return context
+
+    def __len__(self) -> int:
+        """Return number of messages in conversation."""
+        return len(self.messages)
+
+
+@dataclass
+class ConversationTask:
+    """Task for multi-turn conversation execution.
+
+    This extends the basic GenerationTask concept to support
+    multi-turn conversations with configurable stopping conditions.
+
+    Attributes:
+        context: Conversation context with message history
+        model: Model to use for generation
+        sampling: Sampling configuration
+        metadata: Additional metadata
+        reference: Optional reference for evaluation
+        max_turns: Maximum number of conversation turns
+        stop_condition: Optional function to determine when to stop
+    """
+
+    context: ConversationContext
+    model: core_entities.ModelSpec
+    sampling: core_entities.SamplingConfig
+    metadata: dict[str, Any] = field(default_factory=dict)
+    reference: core_entities.Reference | None = None
+    max_turns: int = 10
+    stop_condition: Callable[[ConversationContext], bool] | None = None
+
+    def should_stop(self) -> bool:
+        """Check if conversation should stop.
+
+        Returns:
+            True if stop condition is met or max turns reached
+        """
+        if len(self.context) >= self.max_turns:
+            return True
+
+        if self.stop_condition is not None:
+            return self.stop_condition(self.context)
+
+        return False
+
+
+@dataclass
+class ConversationTurn:
+    """Single turn in a conversation.
+
+    Attributes:
+        turn_number: Turn index (0-based)
+        user_message: User message for this turn (if any)
+        generation_record: Generation result for this turn
+        context_snapshot: Conversation context at this turn
+    """
+
+    turn_number: int
+    user_message: Message | None
+    generation_record: core_entities.GenerationRecord
+    context_snapshot: ConversationContext
+
+
+@dataclass
+class ConversationRecord:
+    """Complete record of a multi-turn conversation.
+
+    This is the result of running a ConversationTask through
+    a ConversationRunner.
+
+    Attributes:
+        task: Original conversation task
+        context: Final conversation context
+        turns: List of turns executed
+        metadata: Additional metadata (e.g., total turns, stop reason)
+    """
+
+    task: ConversationTask
+    context: ConversationContext
+    turns: list[ConversationTurn] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def get_final_output(self) -> core_entities.ModelOutput | None:
+        """Get the final model output.
+
+        Returns:
+            Last turn's output, or None if no turns
+        """
+        if not self.turns:
+            return None
+        return self.turns[-1].generation_record.output
+
+    def get_all_outputs(self) -> list[core_entities.ModelOutput | None]:
+        """Get all model outputs from all turns.
+
+        Returns:
+            List of outputs (may contain None for failed turns)
+        """
+        return [turn.generation_record.output for turn in self.turns]
+
+    def total_turns(self) -> int:
+        """Get total number of turns executed.
+
+        Returns:
+            Number of turns
+        """
+        return len(self.turns)
+
+
+# Common stop conditions
+
+
+def stop_on_keyword(keyword: str) -> Callable[[ConversationContext], bool]:
+    """Create stop condition that triggers when keyword appears.
+
+    Args:
+        keyword: Keyword to look for in assistant messages
+
+    Returns:
+        Stop condition function
+    """
+
+    def condition(context: ConversationContext) -> bool:
+        if not context.messages:
+            return False
+        last_msg = context.messages[-1]
+        if last_msg.role == "assistant":
+            return keyword.lower() in last_msg.content.lower()
+        return False
+
+    return condition
+
+
+def stop_on_pattern(
+    pattern: str,
+) -> Callable[[ConversationContext], bool]:
+    """Create stop condition that triggers when regex pattern matches.
+
+    Args:
+        pattern: Regex pattern to match
+
+    Returns:
+        Stop condition function
+    """
+    import re
+
+    compiled = re.compile(pattern, re.IGNORECASE)
+
+    def condition(context: ConversationContext) -> bool:
+        if not context.messages:
+            return False
+        last_msg = context.messages[-1]
+        if last_msg.role == "assistant":
+            return compiled.search(last_msg.content) is not None
+        return False
+
+    return condition
+
+
+def stop_on_empty_response() -> Callable[[ConversationContext], bool]:
+    """Create stop condition that triggers on empty assistant response.
+
+    Returns:
+        Stop condition function
+    """
+
+    def condition(context: ConversationContext) -> bool:
+        if not context.messages:
+            return False
+        last_msg = context.messages[-1]
+        if last_msg.role == "assistant":
+            return not last_msg.content.strip()
+        return False
+
+    return condition
+
+
+__all__ = [
+    "MessageRole",
+    "Message",
+    "ConversationContext",
+    "ConversationTask",
+    "ConversationTurn",
+    "ConversationRecord",
+    "stop_on_keyword",
+    "stop_on_pattern",
+    "stop_on_empty_response",
+]

themis/core/entities.py

@@ -0,0 +1,184 @@
+"""Shared dataclasses that represent Themis' internal world."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, Generic, List, TypeVar
+
+if TYPE_CHECKING:
+    from themis.evaluation.reports import EvaluationReport
+
+# Type variable for generic Reference
+T = TypeVar("T")
+
+
+@dataclass(frozen=True)
+class SamplingConfig:
+    temperature: float
+    top_p: float
+    max_tokens: int
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    identifier: str
+    provider: str
+    default_sampling: SamplingConfig | None = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class PromptSpec:
+    name: str
+    template: str
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class PromptRender:
+    spec: PromptSpec
+    text: str
+    context: Dict[str, Any] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def prompt_text(self) -> str:
+        return self.text
+
+    @property
+    def template_name(self) -> str:
+        return self.spec.name
+
+
+@dataclass(frozen=True)
+class Reference(Generic[T]):
+    """Reference value with optional type information.
+
+    This is a generic dataclass that can hold typed reference values.
+    For backward compatibility, it can be used without type parameters
+    and will behave like Reference[Any].
+
+    The value field can hold any type including:
+    - Simple types: str, int, float, bool
+    - Collections: list, tuple, set
+    - Dictionaries: dict (for multi-value references)
+    - Custom objects
+
+    Examples:
+        # Simple reference
+        ref = Reference(kind="answer", value="42")
+
+        # Multi-value reference using dict
+        ref = Reference(
+            kind="countdown_task",
+            value={"target": 122, "numbers": [25, 50, 75, 100]}
+        )
+
+        # List reference
+        ref = Reference(kind="valid_answers", value=["yes", "no", "maybe"])
+
+        # Typed reference
+        ref: Reference[str] = Reference(kind="answer", value="42")
+        ref: Reference[dict] = Reference(kind="task", value={"a": 1, "b": 2})
+
+    Note:
+        When using dict values, metrics can access individual fields directly:
+        >>> target = reference.value["target"]
+        >>> numbers = reference.value["numbers"]
+    """
+
+    kind: str
+    value: T
+    schema: type[T] | None = None  # Optional runtime type information
+
+
+@dataclass(frozen=True)
+class ModelOutput:
+    text: str
+    raw: Any | None = None
+    usage: Dict[str, int] | None = None  # Token usage: {prompt_tokens, completion_tokens, total_tokens}
+
+
+@dataclass(frozen=True)
+class ModelError:
+    message: str
+    kind: str = "model_error"
+    details: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class GenerationTask:
+    prompt: PromptRender
+    model: ModelSpec
+    sampling: SamplingConfig
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    reference: Reference | None = None
+
+
+@dataclass
+class GenerationRecord:
+    task: GenerationTask
+    output: ModelOutput | None
+    error: ModelError | None
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    attempts: List["GenerationRecord"] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class EvaluationItem:
+    record: GenerationRecord
+    reference: Reference | None
+
+
+@dataclass(frozen=True)
+class MetricScore:
+    metric_name: str
+    value: float
+    details: Dict[str, Any] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EvaluationSummary:
+    scores: List[MetricScore]
+    failures: List[str] = field(default_factory=list)
+
+
+@dataclass
+class EvaluationRecord:
+    sample_id: str | None
+    scores: List[MetricScore]
+    failures: List[str] = field(default_factory=list)
+
+
+@dataclass
+class ExperimentFailure:
+    sample_id: str | None
+    message: str
+
+
+@dataclass
+class ExperimentReport:
+    generation_results: list[GenerationRecord]
+    evaluation_report: "EvaluationReport"
+    failures: list[ExperimentFailure]
+    metadata: dict[str, object]
+
+
+__all__ = [
+    "SamplingConfig",
+    "ModelSpec",
+    "PromptSpec",
+    "PromptRender",
+    "Reference",
+    "ModelOutput",
+    "ModelError",
+    "GenerationTask",
+    "GenerationRecord",
+    "EvaluationItem",
+    "EvaluationRecord",
+    "MetricScore",
+    "EvaluationSummary",
+    "ExperimentFailure",
+    "ExperimentReport",
+]