DeepFabric 4.4.0__py3-none-any.whl

deepfabric/evaluation/metrics.py

@@ -0,0 +1,397 @@
+"""Metrics computation for model evaluation."""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from ..schemas import ToolDefinition
+
+# Tolerance for numeric comparison
+NUMERIC_TOLERANCE = 1e-6
+
+# Type validation dispatch table
+_TYPE_CHECKS = {
+    "str": lambda v: isinstance(v, str),
+    "int": lambda v: isinstance(v, int) and not isinstance(v, bool),
+    "float": lambda v: isinstance(v, int | float) and not isinstance(v, bool),
+    "bool": lambda v: isinstance(v, bool),
+    "list": lambda v: isinstance(v, list),
+    "dict": lambda v: isinstance(v, dict),
+}
+
+
+def _is_valid_type(schema_type: str, value: Any) -> bool:
+    """Check if value matches schema type.
+
+    Args:
+        schema_type: Schema type string ("str", "int", "float", "bool", "list", "dict")
+        value: Value to check
+
+    Returns:
+        True if value matches type, False otherwise
+    """
+    check = _TYPE_CHECKS.get(schema_type)
+    return check(value) if check else False
+
+
+def _validate_parameter_types(
+    predicted_params: dict[str, Any],
+    tool_def: ToolDefinition,
+) -> bool:
+    """Validate parameter types against tool schema.
+
+    Checks that:
+    1. All required parameters are present
+    2. Parameter types match schema (with type coercion)
+    3. Ignores actual values - only validates structure
+
+    Args:
+        predicted_params: Parameters to validate
+        tool_def: Tool definition with schema
+
+    Returns:
+        True if types are valid, False otherwise
+    """
+    # Create lookup for parameters by name
+    schema_params = {p.name: p for p in tool_def.parameters}
+
+    # Check all required parameters are present
+    for param_name, param_schema in schema_params.items():
+        if param_schema.required and param_name not in predicted_params:
+            return False
+
+    # Check types for each predicted parameter
+    for param_name, predicted_value in predicted_params.items():
+        # Skip extra parameters not in schema (allow for flexibility)
+        if param_name not in schema_params:
+            continue
+
+        schema_param = schema_params[param_name]
+        if not _is_valid_type(schema_param.type, predicted_value):
+            return False
+
+    return True
+
+
+class EvaluationMetrics(BaseModel):
+    """Computed evaluation metrics."""
+
+    tool_selection_accuracy: float = Field(
+        description="Accuracy of tool selection (0.0-1.0)",
+    )
+    parameter_accuracy: float = Field(
+        description="Accuracy of parameter extraction (0.0-1.0)",
+    )
+    execution_success_rate: float = Field(
+        description="Rate of valid tool calls (0.0-1.0)",
+    )
+    response_quality: float = Field(
+        description="Quality of final response (0.0-1.0)",
+    )
+    overall_score: float = Field(
+        description="Weighted overall score (0.0-1.0)",
+    )
+    samples_evaluated: int = Field(
+        description="Total number of samples evaluated",
+    )
+    samples_processed: int = Field(
+        description="Number of samples processed without system errors",
+    )
+    processing_errors: int = Field(
+        description="Number of samples that failed to process (system errors, timeouts)",
+    )
+
+
+class SampleEvaluation(BaseModel):
+    """Evaluation result for a single sample."""
+
+    sample_id: int = Field(description="Sample index")
+    query: str = Field(description="Input query")
+    expected_tool: str | None = Field(
+        default=None,
+        description="Expected tool name",
+    )
+    predicted_tool: str | None = Field(
+        default=None,
+        description="Predicted tool name",
+    )
+    expected_parameters: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Expected parameters",
+    )
+    predicted_parameters: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Predicted parameters",
+    )
+    expected_answer: str | None = Field(
+        default=None,
+        description="Expected final answer",
+    )
+    predicted_answer: str | None = Field(
+        default=None,
+        description="Predicted final answer",
+    )
+    tool_selection_correct: bool = Field(
+        description="Whether tool selection was correct",
+    )
+    parameters_correct: bool = Field(
+        description="Whether parameters were correct",
+    )
+    execution_valid: bool = Field(
+        description="Whether the tool call could be executed",
+    )
+    response_score: float = Field(
+        description="Response quality score (0.0-1.0)",
+    )
+    error: str | None = Field(
+        default=None,
+        description="Error message if prediction failed",
+    )
+
+
+def compute_tool_selection_accuracy(
+    evaluations: list[SampleEvaluation],
+) -> float:
+    """Compute tool selection accuracy.
+
+    Args:
+        evaluations: List of sample evaluations
+
+    Returns:
+        Accuracy score (0.0-1.0)
+    """
+    if not evaluations:
+        return 0.0
+
+    correct = sum(1 for e in evaluations if e.tool_selection_correct)
+    return correct / len(evaluations)
+
+
+def compute_parameter_accuracy(
+    evaluations: list[SampleEvaluation],
+) -> float:
+    """Compute parameter extraction accuracy.
+
+    Args:
+        evaluations: List of sample evaluations
+
+    Returns:
+        Accuracy score (0.0-1.0)
+    """
+    if not evaluations:
+        return 0.0
+
+    correct = sum(1 for e in evaluations if e.parameters_correct)
+    return correct / len(evaluations)
+
+
+def compute_execution_success_rate(
+    evaluations: list[SampleEvaluation],
+) -> float:
+    """Compute execution success rate.
+
+    Args:
+        evaluations: List of sample evaluations
+
+    Returns:
+        Success rate (0.0-1.0)
+    """
+    if not evaluations:
+        return 0.0
+
+    valid = sum(1 for e in evaluations if e.execution_valid)
+    return valid / len(evaluations)
+
+
+def compute_response_quality(
+    evaluations: list[SampleEvaluation],
+) -> float:
+    """Compute average response quality.
+
+    Args:
+        evaluations: List of sample evaluations
+
+    Returns:
+        Average quality score (0.0-1.0)
+    """
+    if not evaluations:
+        return 0.0
+
+    total_score = sum(e.response_score for e in evaluations)
+    return total_score / len(evaluations)
+
+
+def compute_overall_score(
+    tool_accuracy: float,
+    param_accuracy: float,
+    exec_success: float,
+    response_quality: float,
+    weights: dict[str, float] | None = None,
+) -> float:
+    """Compute weighted overall score.
+
+    Args:
+        tool_accuracy: Tool selection accuracy
+        param_accuracy: Parameter accuracy
+        exec_success: Execution success rate
+        response_quality: Response quality score
+        weights: Custom weights for each metric (defaults used if None)
+
+    Returns:
+        Weighted overall score (0.0-1.0)
+    """
+    # Default weights (response_quality excluded for tool-calling mode)
+    if weights is None:
+        weights = {
+            "tool_selection": 0.40,
+            "parameter_accuracy": 0.35,
+            "execution_success": 0.25,
+            "response_quality": 0.00,  # Not used for tool-calling evaluation
+        }
+
+    return (
+        tool_accuracy * weights.get("tool_selection", 0.0)
+        + param_accuracy * weights.get("parameter_accuracy", 0.0)
+        + exec_success * weights.get("execution_success", 0.0)
+        + response_quality * weights.get("response_quality", 0.0)
+    )
+
+
+def compute_metrics(
+    evaluations: list[SampleEvaluation],
+    weights: dict[str, float] | None = None,
+) -> EvaluationMetrics:
+    """Compute all evaluation metrics from sample evaluations.
+
+    Args:
+        evaluations: List of sample evaluations
+        weights: Custom weights for overall score computation
+
+    Returns:
+        EvaluationMetrics with all computed scores
+    """
+    if not evaluations:
+        return EvaluationMetrics(
+            tool_selection_accuracy=0.0,
+            parameter_accuracy=0.0,
+            execution_success_rate=0.0,
+            response_quality=0.0,
+            overall_score=0.0,
+            samples_evaluated=0,
+            samples_processed=0,
+            processing_errors=0,
+        )
+
+    tool_acc = compute_tool_selection_accuracy(evaluations)
+    param_acc = compute_parameter_accuracy(evaluations)
+    exec_success = compute_execution_success_rate(evaluations)
+    resp_quality = compute_response_quality(evaluations)
+
+    overall = compute_overall_score(
+        tool_acc,
+        param_acc,
+        exec_success,
+        resp_quality,
+        weights,
+    )
+
+    # Count processing status (system errors vs successfully processed)
+    processed = sum(1 for e in evaluations if e.error is None)
+    errors = len(evaluations) - processed
+
+    return EvaluationMetrics(
+        tool_selection_accuracy=tool_acc,
+        parameter_accuracy=param_acc,
+        execution_success_rate=exec_success,
+        response_quality=resp_quality,
+        overall_score=overall,
+        samples_evaluated=len(evaluations),
+        samples_processed=processed,
+        processing_errors=errors,
+    )
+
+
+def compare_parameters(  # noqa: PLR0911
+    expected: dict[str, Any],
+    predicted: dict[str, Any],
+    tool_name: str | None = None,
+    tool_definitions: list[ToolDefinition] | None = None,
+) -> bool:
+    """Compare expected and predicted parameters.
+
+    If tool schema is provided, validates parameter types and presence of required params.
+    Otherwise, performs value-based comparison (legacy behavior for backward compatibility).
+
+    Args:
+        expected: Expected parameters
+        predicted: Predicted parameters
+        tool_name: Name of the tool being called (for schema lookup)
+        tool_definitions: List of tool definitions with schemas
+
+    Returns:
+        True if parameters match (schema-aware) or values match (legacy), False otherwise
+    """
+    if not expected and not predicted:
+        return True
+
+    # Schema-aware validation if tool definition available
+    if tool_name and tool_definitions:
+        tool_def = next((t for t in tool_definitions if t.name == tool_name), None)
+        if tool_def:
+            return _validate_parameter_types(predicted, tool_def)
+
+    # Legacy value-based comparison (backward compatibility)
+    # Check if all expected keys are present
+    if set(expected.keys()) != set(predicted.keys()):
+        return False
+
+    # Compare values
+    for key, expected_val in expected.items():
+        predicted_val = predicted.get(key)
+
+        # Handle different types
+        if isinstance(expected_val, str) and isinstance(predicted_val, str):
+            # Case-insensitive string comparison
+            if expected_val.lower().strip() != predicted_val.lower().strip():
+                return False
+        elif isinstance(expected_val, int | float) and isinstance(predicted_val, int | float):
+            # Numeric comparison with small tolerance
+            if abs(float(expected_val) - float(predicted_val)) > NUMERIC_TOLERANCE:
+                return False
+        elif expected_val != predicted_val:
+            # Exact match for other types
+            return False
+
+    return True
+
+
+def compute_response_similarity(
+    expected: str | None,
+    predicted: str | None,
+) -> float:
+    """Compute similarity between expected and predicted responses.
+
+    Uses simple word overlap for now. Can be enhanced with semantic similarity.
+
+    Args:
+        expected: Expected response
+        predicted: Predicted response
+
+    Returns:
+        Similarity score (0.0-1.0)
+    """
+    if not expected or not predicted:
+        return 0.0 if expected != predicted else 1.0
+
+    # Tokenize and normalize
+    expected_words = set(expected.lower().split())
+    predicted_words = set(predicted.lower().split())
+
+    # Compute Jaccard similarity
+    if not expected_words and not predicted_words:
+        return 1.0
+
+    intersection = expected_words & predicted_words
+    union = expected_words | predicted_words
+
+    return len(intersection) / len(union) if union else 0.0

deepfabric/evaluation/parser.py

@@ -0,0 +1,304 @@
+"""Ground truth parsing from DeepFabric dataset samples."""
+
+import json
+import re
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from ..schemas import Conversation, ToolDefinition
+
+
+class ExpectedToolCall(BaseModel):
+    """A single expected tool call with its parameters."""
+
+    tool_name: str = Field(description="Name of the tool")
+    parameters: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Parameter names and values",
+    )
+
+    def signature(self) -> str:
+        """Return a hashable signature for deduplication."""
+        params_str = json.dumps(self.parameters, sort_keys=True)
+        return f"{self.tool_name}:{params_str}"
+
+
+class GroundTruth(BaseModel):
+    """Parsed ground truth from original dataset sample."""
+
+    query: str = Field(description="The user query")
+    expected_tool: str | None = Field(
+        default=None,
+        description="Expected tool name - first tool (None if no tool use). Kept for backwards compatibility.",
+    )
+    expected_parameters: dict[str, str | int | float | bool | list | dict] = Field(
+        default_factory=dict,
+        description="Expected tool parameters - first tool. Kept for backwards compatibility.",
+    )
+    expected_tools: list[ExpectedToolCall] = Field(
+        default_factory=list,
+        description="All unique expected tool calls (deduplicated by tool_name + parameters)",
+    )
+    tool_schema: ToolDefinition | None = Field(
+        default=None,
+        description="Tool schema from available_tools",
+    )
+    expected_answer: str | None = Field(
+        default=None,
+        description="Expected final answer if available",
+    )
+    conversation_type: Literal["basic", "chain_of_thought"] = Field(
+        description="Type of conversation",
+    )
+    reasoning_style: Literal["freetext", "agent", "structured", "hybrid"] | None = Field(
+        default=None,
+        description="Reasoning style if chain_of_thought",
+    )
+    agent_mode: Literal["single_turn", "multi_turn"] | None = Field(
+        default=None,
+        description="Agent mode if tools are used",
+    )
+    metadata: dict[str, str | int | float | bool] = Field(
+        default_factory=dict,
+        description="Additional metadata",
+    )
+
+
+class GroundTruthParser:
+    """Parse ground truth from original DeepFabric JSONL format.
+
+    This parser extracts expected tools, parameters, and answers from
+    Conversation objects while handling all conversation types and agent modes.
+    """
+
+    def __init__(
+        self,
+        conversation_type: Literal["basic", "chain_of_thought"],
+        reasoning_style: Literal["freetext", "agent", "structured", "hybrid"] | None = None,
+        agent_mode: Literal["single_turn", "multi_turn"] | None = None,
+    ):
+        """Initialize parser with conversation configuration.
+
+        Args:
+            conversation_type: Type of conversation (basic, chain_of_thought)
+            reasoning_style: Reasoning style for chain_of_thought
+            agent_mode: Agent mode if tools are used
+        """
+        self.conversation_type: Literal["basic", "chain_of_thought"] = conversation_type
+        self.reasoning_style: Literal["freetext", "agent", "structured", "hybrid"] | None = (
+            reasoning_style
+        )
+        self.agent_mode: Literal["single_turn", "multi_turn"] | None = agent_mode
+
+    def parse(self, conversation: Conversation) -> GroundTruth:
+        """Extract ground truth from a conversation sample.
+
+        Args:
+            conversation: Conversation object from dataset
+
+        Returns:
+            GroundTruth with expected values
+
+        Raises:
+            ValueError: If conversation format is invalid
+        """
+        # Extract query from first user message
+        query = self._extract_query(conversation)
+
+        # Extract expected tool and parameters if tool_context present
+        expected_tool: str | None = None
+        expected_parameters: dict = {}
+        expected_tools: list[ExpectedToolCall] = []
+        tool_schema: ToolDefinition | None = None
+
+        executions = (
+            conversation.tool_context.executions
+            if conversation.tool_context is not None and conversation.tool_context.executions
+            else []
+        )
+        if executions:
+            # Get first tool execution for backwards compatibility
+            first_execution = executions[0]
+            expected_tool = first_execution.function_name
+            expected_parameters = first_execution.parsed_arguments
+
+            # Extract ALL tool executions and deduplicate
+            seen_signatures: set[str] = set()
+            for execution in executions:
+                tool_call = ExpectedToolCall(
+                    tool_name=execution.function_name,
+                    parameters=execution.parsed_arguments,
+                )
+                sig = tool_call.signature()
+                if sig not in seen_signatures:
+                    seen_signatures.add(sig)
+                    expected_tools.append(tool_call)
+
+            # Get tool schema from tools field (OpenAI format)
+            if conversation.tools:
+                available_tools = [ToolDefinition.from_openai(tool) for tool in conversation.tools]
+                tool_schema = self._get_tool_schema(available_tools, expected_tool)
+
+        # Extract expected answer
+        expected_answer = self._extract_expected_answer(conversation)
+
+        # Extract metadata
+        metadata_dict: dict[str, str | int | float | bool] = {}
+        if conversation.metadata:
+            # Filter to only simple types
+            for key, value in conversation.metadata.items():
+                if isinstance(value, str | int | float | bool):
+                    metadata_dict[key] = value
+
+        return GroundTruth(
+            query=query,
+            expected_tool=expected_tool,
+            expected_parameters=expected_parameters,
+            expected_tools=expected_tools,
+            tool_schema=tool_schema,
+            expected_answer=expected_answer,
+            conversation_type=self.conversation_type,
+            reasoning_style=self.reasoning_style,
+            agent_mode=self.agent_mode,
+            metadata=metadata_dict,
+        )
+
+    def _extract_query(self, conversation: Conversation) -> str:
+        """Extract user query from conversation messages.
+
+        Args:
+            conversation: Conversation object
+
+        Returns:
+            User query string
+
+        Raises:
+            ValueError: If no user message found
+        """
+        # Find first user message
+        for message in conversation.messages:
+            if message.role == "user":
+                return message.content
+
+        # Fallback to question field if present
+        if conversation.question:
+            return conversation.question
+
+        raise ValueError("No user query found in conversation")
+
+    def _get_tool_schema(
+        self,
+        available_tools: list[ToolDefinition],
+        tool_name: str,
+    ) -> ToolDefinition | None:
+        """Get tool schema by name from available tools.
+
+        Args:
+            available_tools: List of available tool definitions
+            tool_name: Name of tool to find
+
+        Returns:
+            ToolDefinition if found, None otherwise
+        """
+        for tool in available_tools:
+            if tool.name == tool_name:
+                return tool
+        return None
+
+    def _extract_expected_answer(self, conversation: Conversation) -> str | None:
+        """Extract expected answer from conversation.
+
+        Args:
+            conversation: Conversation object
+
+        Returns:
+            Expected answer if available, None otherwise
+        """
+        # Check final_answer field first
+        if conversation.final_answer:
+            return conversation.final_answer
+
+        # For tool-calling conversations, answer is in last assistant message
+        # after tool execution
+        if conversation.tool_context:
+            # Find last assistant message
+            for message in reversed(conversation.messages):
+                if message.role == "assistant" and not self._contains_tool_call(message.content):
+                    # Skip messages that contain tool calls
+                    return message.content
+
+        # For basic conversations, last assistant message is the answer
+        for message in reversed(conversation.messages):
+            if message.role == "assistant":
+                return message.content
+
+        return None
+
+    def _contains_tool_call(self, content: str) -> bool:
+        """Check if message content contains a tool call.
+
+        Looks for common tool call patterns:
+        - XML: <tool_call>...</tool_call>
+        - JSON: {"tool_calls": ...}
+        - Function: function_name(...)
+
+        Args:
+            content: Message content
+
+        Returns:
+            True if tool call detected
+        """
+        # Check for XML tool call tags
+        if "<tool_call>" in content or "</tool_call>" in content:
+            return True
+
+        # Check for JSON tool calls
+        if "{" in content and "tool_calls" in content:
+            try:
+                data = json.loads(content)
+                if "tool_calls" in data or "function_call" in data:
+                    return True
+            except json.JSONDecodeError:
+                pass
+
+        # Check for function call pattern: func_name(arg1, arg2)
+        func_pattern = r"\b[a-z_][a-z0-9_]*\s*\([^)]*\)"
+        return bool(re.search(func_pattern, content.lower()))
+
+
+def parse_batch(
+    conversations: list[Conversation],
+    conversation_type: Literal["basic", "chain_of_thought"],
+    reasoning_style: Literal["freetext", "agent", "structured", "hybrid"] | None = None,
+    agent_mode: Literal["single_turn", "multi_turn"] | None = None,
+) -> list[GroundTruth]:
+    """Parse a batch of conversations to extract ground truth.
+
+    Args:
+        conversations: List of Conversation objects
+        conversation_type: Type of conversation
+        reasoning_style: Reasoning style if chain_of_thought
+        agent_mode: Agent mode if tools are used
+
+    Returns:
+        List of GroundTruth objects
+    """
+    parser = GroundTruthParser(
+        conversation_type=conversation_type,
+        reasoning_style=reasoning_style,
+        agent_mode=agent_mode,
+    )
+
+    ground_truths: list[GroundTruth] = []
+    for conversation in conversations:
+        try:
+            gt = parser.parse(conversation)
+            ground_truths.append(gt)
+        except ValueError as e:
+            # Log error but continue processing
+            print(f"Warning: Failed to parse conversation: {e}")
+            continue
+
+    return ground_truths

deepfabric/evaluation/reporters/__init__.py

@@ -0,0 +1,13 @@
+"""Reporters for evaluation result output."""
+
+from .base import BaseReporter
+from .cloud_reporter import CloudReporter
+from .file_reporter import FileReporter
+from .multi_reporter import MultiReporter
+
+__all__ = [
+    "BaseReporter",
+    "FileReporter",
+    "CloudReporter",
+    "MultiReporter",
+]