themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/core/serialization.py

@@ -0,0 +1,231 @@
+"""Serialization helpers for Themis core entities."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Dict
+
+from themis.core import entities as core_entities
+
+
+def serialize_sampling(config: core_entities.SamplingConfig) -> Dict[str, Any]:
+    return {
+        "temperature": config.temperature,
+        "top_p": config.top_p,
+        "max_tokens": config.max_tokens,
+    }
+
+
+def deserialize_sampling(data: Dict[str, Any]) -> core_entities.SamplingConfig:
+    return core_entities.SamplingConfig(
+        temperature=data["temperature"],
+        top_p=data["top_p"],
+        max_tokens=data["max_tokens"],
+    )
+
+
+def serialize_model_spec(spec: core_entities.ModelSpec) -> Dict[str, Any]:
+    return {
+        "identifier": spec.identifier,
+        "provider": spec.provider,
+        "metadata": copy.deepcopy(spec.metadata),
+        "default_sampling": serialize_sampling(spec.default_sampling)
+        if spec.default_sampling
+        else None,
+    }
+
+
+def deserialize_model_spec(data: Dict[str, Any]) -> core_entities.ModelSpec:
+    default_sampling = (
+        deserialize_sampling(data["default_sampling"])
+        if data.get("default_sampling")
+        else None
+    )
+    return core_entities.ModelSpec(
+        identifier=data["identifier"],
+        provider=data["provider"],
+        metadata=copy.deepcopy(data.get("metadata", {})),
+        default_sampling=default_sampling,
+    )
+
+
+def serialize_prompt_spec(spec: core_entities.PromptSpec) -> Dict[str, Any]:
+    return {
+        "name": spec.name,
+        "template": spec.template,
+        "metadata": copy.deepcopy(spec.metadata),
+    }
+
+
+def deserialize_prompt_spec(data: Dict[str, Any]) -> core_entities.PromptSpec:
+    return core_entities.PromptSpec(
+        name=data["name"],
+        template=data["template"],
+        metadata=copy.deepcopy(data.get("metadata", {})),
+    )
+
+
+def serialize_prompt_render(render: core_entities.PromptRender) -> Dict[str, Any]:
+    return {
+        "spec": serialize_prompt_spec(render.spec),
+        "text": render.text,
+        "context": copy.deepcopy(render.context),
+        "metadata": copy.deepcopy(render.metadata),
+    }
+
+
+def deserialize_prompt_render(data: Dict[str, Any]) -> core_entities.PromptRender:
+    return core_entities.PromptRender(
+        spec=deserialize_prompt_spec(data["spec"]),
+        text=data["text"],
+        context=copy.deepcopy(data.get("context", {})),
+        metadata=copy.deepcopy(data.get("metadata", {})),
+    )
+
+
+def serialize_reference(
+    reference: core_entities.Reference | None,
+) -> Dict[str, Any] | None:
+    if reference is None:
+        return None
+    return {"kind": reference.kind, "value": reference.value}
+
+
+def deserialize_reference(
+    data: Dict[str, Any] | None,
+) -> core_entities.Reference | None:
+    if data is None:
+        return None
+    return core_entities.Reference(kind=data["kind"], value=data.get("value"))
+
+
+def serialize_generation_task(task: core_entities.GenerationTask) -> Dict[str, Any]:
+    return {
+        "prompt": serialize_prompt_render(task.prompt),
+        "model": serialize_model_spec(task.model),
+        "sampling": serialize_sampling(task.sampling),
+        "metadata": copy.deepcopy(task.metadata),
+        "reference": serialize_reference(task.reference),
+    }
+
+
+def deserialize_generation_task(data: Dict[str, Any]) -> core_entities.GenerationTask:
+    return core_entities.GenerationTask(
+        prompt=deserialize_prompt_render(data["prompt"]),
+        model=deserialize_model_spec(data["model"]),
+        sampling=deserialize_sampling(data["sampling"]),
+        metadata=copy.deepcopy(data.get("metadata", {})),
+        reference=deserialize_reference(data.get("reference")),
+    )
+
+
+def serialize_generation_record(
+    record: core_entities.GenerationRecord,
+) -> Dict[str, Any]:
+    return {
+        "task": serialize_generation_task(record.task),
+        "output": {
+            "text": record.output.text,
+            "raw": record.output.raw,
+        }
+        if record.output
+        else None,
+        "error": {
+            "message": record.error.message,
+            "kind": record.error.kind,
+            "details": copy.deepcopy(record.error.details),
+        }
+        if record.error
+        else None,
+        "metrics": copy.deepcopy(record.metrics),
+        "attempts": [
+            serialize_generation_record(attempt) for attempt in record.attempts
+        ],
+    }
+
+
+def deserialize_generation_record(
+    data: Dict[str, Any],
+) -> core_entities.GenerationRecord:
+    output_data = data.get("output")
+    error_data = data.get("error")
+    return core_entities.GenerationRecord(
+        task=deserialize_generation_task(data["task"]),
+        output=core_entities.ModelOutput(
+            text=output_data["text"], raw=output_data.get("raw")
+        )
+        if output_data
+        else None,
+        error=core_entities.ModelError(
+            message=error_data["message"],
+            kind=error_data.get("kind", "model_error"),
+            details=copy.deepcopy(error_data.get("details", {})),
+        )
+        if error_data
+        else None,
+        metrics=copy.deepcopy(data.get("metrics", {})),
+        attempts=[
+            deserialize_generation_record(attempt)
+            for attempt in data.get("attempts", [])
+        ],
+    )
+
+
+def serialize_metric_score(score: core_entities.MetricScore) -> Dict[str, Any]:
+    return {
+        "metric_name": score.metric_name,
+        "value": score.value,
+        "details": copy.deepcopy(score.details),
+        "metadata": copy.deepcopy(score.metadata),
+    }
+
+
+def deserialize_metric_score(data: Dict[str, Any]) -> core_entities.MetricScore:
+    return core_entities.MetricScore(
+        metric_name=data["metric_name"],
+        value=data["value"],
+        details=copy.deepcopy(data.get("details", {})),
+        metadata=copy.deepcopy(data.get("metadata", {})),
+    )
+
+
+def serialize_evaluation_record(
+    record: core_entities.EvaluationRecord,
+) -> Dict[str, Any]:
+    return {
+        "sample_id": record.sample_id,
+        "scores": [serialize_metric_score(score) for score in record.scores],
+        "failures": list(record.failures),
+    }
+
+
+def deserialize_evaluation_record(
+    data: Dict[str, Any],
+) -> core_entities.EvaluationRecord:
+    return core_entities.EvaluationRecord(
+        sample_id=data.get("sample_id"),
+        scores=[deserialize_metric_score(score) for score in data.get("scores", [])],
+        failures=list(data.get("failures", [])),
+    )
+
+
+__all__ = [
+    "serialize_generation_record",
+    "deserialize_generation_record",
+    "serialize_generation_task",
+    "deserialize_generation_task",
+    "serialize_evaluation_record",
+    "deserialize_evaluation_record",
+    "serialize_metric_score",
+    "deserialize_metric_score",
+    "serialize_sampling",
+    "deserialize_sampling",
+    "serialize_model_spec",
+    "deserialize_model_spec",
+    "serialize_prompt_spec",
+    "deserialize_prompt_spec",
+    "serialize_prompt_render",
+    "deserialize_prompt_render",
+    "serialize_reference",
+    "deserialize_reference",
+]

themis/core/tools.py

@@ -0,0 +1,393 @@
+"""Tool use primitives for agentic workflows.
+
+This module provides abstractions for defining and executing tools
+(functions) that models can call during generation. This enables
+agentic workflows, function calling, and tool-augmented generation.
+
+Examples:
+    # Define a tool
+    def calculator(operation: str, a: float, b: float) -> float:
+        if operation == "add":
+            return a + b
+        elif operation == "multiply":
+            return a * b
+        raise ValueError(f"Unknown operation: {operation}")
+
+    tool = ToolDefinition(
+        name="calculator",
+        description="Perform arithmetic operations",
+        parameters={
+            "type": "object",
+            "properties": {
+                "operation": {"type": "string", "enum": ["add", "multiply"]},
+                "a": {"type": "number"},
+                "b": {"type": "number"},
+            },
+            "required": ["operation", "a", "b"],
+        },
+        handler=calculator
+    )
+
+    # Register tool
+    registry = ToolRegistry()
+    registry.register(tool)
+
+    # Execute tool
+    call = ToolCall(tool_name="calculator", arguments={"operation": "add", "a": 2, "b": 3})
+    result = registry.execute(call)
+    print(result.result)  # 5.0
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+
+@dataclass
+class ToolDefinition:
+    """Defines a tool/function available to the model.
+
+    Attributes:
+        name: Tool name (should be unique)
+        description: Human-readable description of what tool does
+        parameters: JSON Schema describing parameters
+        handler: Function to execute when tool is called
+        metadata: Additional metadata
+    """
+
+    name: str
+    description: str
+    parameters: dict[str, Any]
+    handler: Callable[[dict[str, Any]], Any]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert tool definition to dictionary (without handler).
+
+        Returns:
+            Dictionary representation suitable for JSON serialization
+        """
+        return {
+            "name": self.name,
+            "description": self.description,
+            "parameters": self.parameters,
+            "metadata": self.metadata,
+        }
+
+    def validate_arguments(self, arguments: dict[str, Any]) -> list[str]:
+        """Validate arguments against parameter schema.
+
+        Args:
+            arguments: Arguments to validate
+
+        Returns:
+            List of validation error messages (empty if valid)
+        """
+        errors = []
+
+        # Simple validation - check required fields
+        if "required" in self.parameters:
+            for field in self.parameters["required"]:
+                if field not in arguments:
+                    errors.append(f"Missing required field: {field}")
+
+        # Check for unknown fields
+        if "properties" in self.parameters:
+            known_fields = set(self.parameters["properties"].keys())
+            for field in arguments.keys():
+                if field not in known_fields:
+                    errors.append(f"Unknown field: {field}")
+
+        return errors
+
+
+@dataclass
+class ToolCall:
+    """Represents a request to execute a tool.
+
+    Attributes:
+        tool_name: Name of tool to execute
+        arguments: Arguments to pass to tool
+        call_id: Unique identifier for this call
+    """
+
+    tool_name: str
+    arguments: dict[str, Any]
+    call_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary.
+
+        Returns:
+            Dictionary representation
+        """
+        return {
+            "tool_name": self.tool_name,
+            "arguments": self.arguments,
+            "call_id": self.call_id,
+        }
+
+
+@dataclass
+class ToolResult:
+    """Result from executing a tool.
+
+    Attributes:
+        call: Original tool call
+        result: Result value (if successful)
+        error: Error message (if failed)
+        execution_time_ms: Time taken to execute (milliseconds)
+        metadata: Additional metadata
+    """
+
+    call: ToolCall
+    result: Any | None
+    error: str | None
+    execution_time_ms: float
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def is_success(self) -> bool:
+        """Check if tool execution was successful.
+
+        Returns:
+            True if no error
+        """
+        return self.error is None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary.
+
+        Returns:
+            Dictionary representation
+        """
+        return {
+            "call": self.call.to_dict(),
+            "result": self.result,
+            "error": self.error,
+            "execution_time_ms": self.execution_time_ms,
+            "metadata": self.metadata,
+        }
+
+
+class ToolRegistry:
+    """Registry for managing and executing tools.
+
+    This class maintains a registry of available tools and provides
+    methods for registering, retrieving, and executing them.
+
+    Examples:
+        registry = ToolRegistry()
+
+        # Register tools
+        registry.register(calculator_tool)
+        registry.register(search_tool)
+
+        # Execute tool
+        call = ToolCall(tool_name="calculator", arguments={...})
+        result = registry.execute(call)
+    """
+
+    def __init__(self):
+        """Initialize empty tool registry."""
+        self._tools: dict[str, ToolDefinition] = {}
+
+    def register(self, tool: ToolDefinition) -> None:
+        """Register a tool.
+
+        Args:
+            tool: Tool definition to register
+
+        Raises:
+            ValueError: If tool with same name already registered
+        """
+        if tool.name in self._tools:
+            raise ValueError(f"Tool '{tool.name}' already registered")
+
+        self._tools[tool.name] = tool
+
+    def unregister(self, name: str) -> None:
+        """Unregister a tool by name.
+
+        Args:
+            name: Tool name to unregister
+        """
+        self._tools.pop(name, None)
+
+    def get(self, name: str) -> ToolDefinition | None:
+        """Get tool by name.
+
+        Args:
+            name: Tool name
+
+        Returns:
+            ToolDefinition if found, None otherwise
+        """
+        return self._tools.get(name)
+
+    def list_tools(self) -> list[ToolDefinition]:
+        """Get all registered tools.
+
+        Returns:
+            List of tool definitions
+        """
+        return list(self._tools.values())
+
+    def execute(self, call: ToolCall) -> ToolResult:
+        """Execute a tool call.
+
+        Args:
+            call: Tool call to execute
+
+        Returns:
+            ToolResult with execution result or error
+        """
+        tool = self._tools.get(call.tool_name)
+
+        if tool is None:
+            return ToolResult(
+                call=call,
+                result=None,
+                error=f"Unknown tool: {call.tool_name}",
+                execution_time_ms=0.0,
+            )
+
+        # Validate arguments
+        validation_errors = tool.validate_arguments(call.arguments)
+        if validation_errors:
+            return ToolResult(
+                call=call,
+                result=None,
+                error=f"Invalid arguments: {'; '.join(validation_errors)}",
+                execution_time_ms=0.0,
+            )
+
+        # Execute tool
+        start = time.perf_counter()
+        try:
+            result = tool.handler(call.arguments)
+            elapsed = (time.perf_counter() - start) * 1000
+            return ToolResult(
+                call=call,
+                result=result,
+                error=None,
+                execution_time_ms=elapsed,
+            )
+        except Exception as e:
+            elapsed = (time.perf_counter() - start) * 1000
+            return ToolResult(
+                call=call,
+                result=None,
+                error=f"{e.__class__.__name__}: {str(e)}",
+                execution_time_ms=elapsed,
+            )
+
+    def to_dict_list(self) -> list[dict[str, Any]]:
+        """Get all tools as dictionary list (for sending to model).
+
+        Returns:
+            List of tool definitions as dictionaries
+        """
+        return [tool.to_dict() for tool in self._tools.values()]
+
+
+# Built-in tools for common use cases
+
+
+def create_calculator_tool() -> ToolDefinition:
+    """Create a basic calculator tool.
+
+    Returns:
+        ToolDefinition for calculator
+    """
+
+    def handler(args: dict[str, Any]) -> float:
+        operation = args["operation"]
+        a = float(args["a"])
+        b = float(args["b"])
+
+        if operation == "add":
+            return a + b
+        elif operation == "subtract":
+            return a - b
+        elif operation == "multiply":
+            return a * b
+        elif operation == "divide":
+            if b == 0:
+                raise ValueError("Division by zero")
+            return a / b
+        else:
+            raise ValueError(f"Unknown operation: {operation}")
+
+    return ToolDefinition(
+        name="calculator",
+        description="Perform basic arithmetic operations (add, subtract, multiply, divide)",
+        parameters={
+            "type": "object",
+            "properties": {
+                "operation": {
+                    "type": "string",
+                    "enum": ["add", "subtract", "multiply", "divide"],
+                    "description": "The arithmetic operation to perform",
+                },
+                "a": {"type": "number", "description": "First number"},
+                "b": {"type": "number", "description": "Second number"},
+            },
+            "required": ["operation", "a", "b"],
+        },
+        handler=handler,
+    )
+
+
+def create_counter_tool() -> ToolDefinition:
+    """Create a stateful counter tool for testing.
+
+    Returns:
+        ToolDefinition for counter
+    """
+    counter = {"value": 0}
+
+    def handler(args: dict[str, Any]) -> int:
+        action = args["action"]
+
+        if action == "increment":
+            counter["value"] += 1
+        elif action == "decrement":
+            counter["value"] -= 1
+        elif action == "reset":
+            counter["value"] = 0
+        elif action == "get":
+            pass  # Just return current value
+        else:
+            raise ValueError(f"Unknown action: {action}")
+
+        return counter["value"]
+
+    return ToolDefinition(
+        name="counter",
+        description="Simple counter that can be incremented, decremented, or reset",
+        parameters={
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": ["increment", "decrement", "reset", "get"],
+                    "description": "Action to perform on counter",
+                },
+            },
+            "required": ["action"],
+        },
+        handler=handler,
+    )
+
+
+__all__ = [
+    "ToolDefinition",
+    "ToolCall",
+    "ToolResult",
+    "ToolRegistry",
+    "create_calculator_tool",
+    "create_counter_tool",
+]