multivon-eval 0.1.0__py3-none-any.whl

multivon_eval/__init__.py

@@ -0,0 +1,56 @@
+"""
+multivon-eval — AI evaluation for teams that ship models to production.
+
+Evaluate language models, agents, and pipelines across text, RAG,
+agentic workflows, and multi-turn conversations.
+
+Quick start:
+    from multivon_eval import EvalSuite, EvalCase, Relevance, Faithfulness, NotEmpty
+
+    suite = EvalSuite("My Eval")
+    suite.add_cases([EvalCase(input="What is 2+2?", expected_output="4")])
+    suite.add_evaluators(NotEmpty(), Relevance())
+    report = suite.run(my_model_fn)
+"""
+
+__version__ = "0.1.0"
+
+from .suite import EvalSuite
+from .case import EvalCase, AgentStep, ToolCall
+from .dataset import load, load_jsonl, load_csv
+from .evaluators import (
+    # Deterministic
+    NotEmpty, ExactMatch, Contains, RegexMatch,
+    JSONSchemaEval, WordCount, Latency, MaxLatency,
+    BLEU, ROUGE, StartsWith,
+    # LLM-as-judge (QAG)
+    Faithfulness, Hallucination, Relevance, Coherence,
+    Toxicity, Bias, Summarization, AnswerAccuracy,
+    ContextPrecision, ContextRecall, CustomRubric, GEval,
+    # Agent
+    ToolCallAccuracy, ToolArgumentAccuracy,
+    PlanQuality, TaskCompletion, StepFaithfulness,
+    # Conversation
+    ConversationRelevance, KnowledgeRetention,
+    ConversationCompleteness, TurnConsistency,
+)
+
+__all__ = [
+    "__version__",
+    "EvalSuite", "EvalCase", "AgentStep", "ToolCall",
+    "load", "load_jsonl", "load_csv",
+    # Deterministic
+    "NotEmpty", "ExactMatch", "Contains", "RegexMatch",
+    "JSONSchemaEval", "WordCount", "Latency", "MaxLatency",
+    "BLEU", "ROUGE", "StartsWith",
+    # LLM-as-judge
+    "Faithfulness", "Hallucination", "Relevance", "Coherence",
+    "Toxicity", "Bias", "Summarization", "AnswerAccuracy",
+    "ContextPrecision", "ContextRecall", "CustomRubric", "GEval",
+    # Agent
+    "ToolCallAccuracy", "ToolArgumentAccuracy",
+    "PlanQuality", "TaskCompletion", "StepFaithfulness",
+    # Conversation
+    "ConversationRelevance", "KnowledgeRetention",
+    "ConversationCompleteness", "TurnConsistency",
+]

multivon_eval/case.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ToolCall:
+    """A single tool/function call made by an agent."""
+    name: str
+    arguments: dict[str, Any] = field(default_factory=dict)
+    result: Any = None
+
+
+@dataclass
+class AgentStep:
+    """One step in an agent's execution trace."""
+    thought: str = ""
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    output: str = ""
+
+
+@dataclass
+class EvalCase:
+    """
+    A single test case for evaluation.
+
+    Attributes:
+        input:                The prompt, question, or user message.
+        expected_output:      Ideal response (used by ExactMatch, AnswerAccuracy).
+        context:              Retrieved documents or context (used by Faithfulness, Hallucination).
+        conversation:         Multi-turn message history for conversation evaluators.
+                              Format: [{"role": "user"/"assistant", "content": "..."}]
+        agent_trace:          Sequence of agent steps for agent evaluators.
+        expected_tool_calls:  Ordered list of tool names the agent should call.
+        metadata:             Arbitrary key-value data (e.g. source_id, difficulty).
+        tags:                 Labels for filtering reports.
+    """
+    input: str
+    expected_output: str | None = None
+    context: str | list[str] | None = None
+    conversation: list[dict[str, str]] | None = None
+    agent_trace: list[AgentStep] | None = None
+    expected_tool_calls: list[str] | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    tags: list[str] = field(default_factory=list)
+
+    def context_str(self) -> str:
+        if self.context is None:
+            return ""
+        if isinstance(self.context, list):
+            return "\n\n".join(self.context)
+        return self.context
+
+    def conversation_str(self) -> str:
+        if not self.conversation:
+            return ""
+        return "\n".join(f"{m['role'].upper()}: {m['content']}" for m in self.conversation)

multivon_eval/cli.py

@@ -0,0 +1,72 @@
+"""
+multivon-eval CLI — run eval files and view reports.
+
+Usage:
+    multivon-eval run eval.py
+    multivon-eval report results.json
+"""
+from __future__ import annotations
+import sys
+import json
+import argparse
+
+
+def cmd_run(args):
+    """Execute a Python eval file."""
+    import runpy
+    runpy.run_path(args.file, run_name="__main__")
+
+
+def cmd_report(args):
+    """Pretty-print a saved JSON report."""
+    from rich.console import Console
+    from rich.table import Table
+    from rich import box
+
+    console = Console()
+    with open(args.file) as f:
+        data = json.load(f)
+
+    console.rule(f"[bold]{data.get('suite', 'Eval Report')}[/]")
+    if data.get("model"):
+        console.print(f"  Model: [dim]{data['model']}[/]")
+
+    summary = data.get("summary", {})
+    console.print(f"\n  Total: {summary.get('total')}  "
+                  f"Passed: [green]{summary.get('passed')}[/]  "
+                  f"Failed: [red]{summary.get('failed')}[/]  "
+                  f"Pass Rate: {summary.get('pass_rate', 0):.1%}\n")
+
+    by_ev = summary.get("by_evaluator", {})
+    if by_ev:
+        t = Table(box=box.SIMPLE_HEAD, title="By Evaluator", padding=(0, 1))
+        t.add_column("Evaluator")
+        t.add_column("Avg Score", justify="right")
+        for name, score in by_ev.items():
+            color = "green" if score >= 0.7 else "yellow" if score >= 0.5 else "red"
+            t.add_row(name, f"[{color}]{score:.3f}[/]")
+        console.print(t)
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="multivon-eval", description="Multivon Eval CLI")
+    sub = parser.add_subparsers(dest="command")
+
+    run_p = sub.add_parser("run", help="Execute an eval file")
+    run_p.add_argument("file", help="Python eval file to run")
+
+    report_p = sub.add_parser("report", help="Display a saved JSON report")
+    report_p.add_argument("file", help="JSON results file")
+
+    args = parser.parse_args()
+    if args.command == "run":
+        cmd_run(args)
+    elif args.command == "report":
+        cmd_report(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

multivon_eval/dataset.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+import csv
+import json
+from pathlib import Path
+from .case import EvalCase
+
+
+def load_jsonl(path: str) -> list[EvalCase]:
+    """Load test cases from a JSONL file. Each line is a JSON object."""
+    cases = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            data = json.loads(line)
+            cases.append(EvalCase(
+                input=data["input"],
+                expected_output=data.get("expected_output"),
+                context=data.get("context"),
+                metadata=data.get("metadata", {}),
+                tags=data.get("tags", []),
+            ))
+    return cases
+
+
+def load_csv(path: str) -> list[EvalCase]:
+    """Load test cases from a CSV file with columns: input, expected_output, context, tags."""
+    cases = []
+    with open(path, newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            tags = [t.strip() for t in row.get("tags", "").split(",") if t.strip()]
+            cases.append(EvalCase(
+                input=row["input"],
+                expected_output=row.get("expected_output") or None,
+                context=row.get("context") or None,
+                tags=tags,
+            ))
+    return cases
+
+
+def load(path: str) -> list[EvalCase]:
+    """Auto-detect format from file extension and load cases."""
+    p = Path(path)
+    if p.suffix == ".jsonl":
+        return load_jsonl(path)
+    elif p.suffix == ".csv":
+        return load_csv(path)
+    raise ValueError(f"Unsupported format: {p.suffix}. Use .jsonl or .csv")

multivon_eval/evaluators/__init__.py

@@ -0,0 +1,35 @@
+from .deterministic import (
+    NotEmpty, ExactMatch, Contains, RegexMatch,
+    JSONSchemaEval, WordCount, Latency, MaxLatency,
+    BLEU, ROUGE, StartsWith,
+)
+from .llm_judge import (
+    Faithfulness, Hallucination, Relevance, Coherence,
+    Toxicity, Bias, Summarization, AnswerAccuracy,
+    ContextPrecision, ContextRecall, CustomRubric, GEval,
+)
+from .agent import (
+    ToolCallAccuracy, ToolArgumentAccuracy,
+    PlanQuality, TaskCompletion, StepFaithfulness,
+)
+from .conversation import (
+    ConversationRelevance, KnowledgeRetention,
+    ConversationCompleteness, TurnConsistency,
+)
+
+__all__ = [
+    # Deterministic
+    "NotEmpty", "ExactMatch", "Contains", "RegexMatch",
+    "JSONSchemaEval", "WordCount", "Latency", "MaxLatency",
+    "BLEU", "ROUGE", "StartsWith",
+    # LLM-as-judge
+    "Faithfulness", "Hallucination", "Relevance", "Coherence",
+    "Toxicity", "Bias", "Summarization", "AnswerAccuracy",
+    "ContextPrecision", "ContextRecall", "CustomRubric", "GEval",
+    # Agent
+    "ToolCallAccuracy", "ToolArgumentAccuracy",
+    "PlanQuality", "TaskCompletion", "StepFaithfulness",
+    # Conversation
+    "ConversationRelevance", "KnowledgeRetention",
+    "ConversationCompleteness", "TurnConsistency",
+]

multivon_eval/evaluators/agent.py

@@ -0,0 +1,223 @@
+"""
+Agent evaluators — evaluate multi-step AI agent execution traces.
+
+These evaluators operate on AgentStep traces attached to EvalCase,
+not just the final output string. This is the key differentiator:
+framework-agnostic evaluation of tool use, planning, and task completion.
+"""
+from __future__ import annotations
+import json
+import re
+
+from .base import Evaluator
+from .llm_judge import _judge_call, _parse_yes_no, _qag_eval
+from ..case import EvalCase, AgentStep
+from ..result import EvalResult
+
+
+def _trace_str(trace: list[AgentStep]) -> str:
+    """Render an agent trace as readable text for the judge."""
+    lines = []
+    for i, step in enumerate(trace, 1):
+        lines.append(f"Step {i}:")
+        if step.thought:
+            lines.append(f"  Thought: {step.thought}")
+        for tc in step.tool_calls:
+            args = json.dumps(tc.arguments, indent=2) if tc.arguments else "{}"
+            result_str = str(tc.result)[:200] if tc.result is not None else "(no result)"
+            lines.append(f"  Tool call: {tc.name}({args})")
+            lines.append(f"  Result: {result_str}")
+        if step.output:
+            lines.append(f"  Output: {step.output}")
+    return "\n".join(lines)
+
+
+class ToolCallAccuracy(Evaluator):
+    """
+    Evaluates whether the agent called the right tools in the right order.
+
+    Checks:
+    - Were all expected tools called?
+    - Were they called in the correct order (if order matters)?
+    - Were any unexpected tools called?
+
+    Requires case.agent_trace and case.expected_tool_calls.
+    """
+    name = "tool_call_accuracy"
+
+    def __init__(self, require_order: bool = False, threshold: float = 0.7):
+        super().__init__(threshold)
+        self.require_order = require_order
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.agent_trace:
+            return self._result(0.0, "No agent_trace provided")
+        if not case.expected_tool_calls:
+            return self._result(0.0, "No expected_tool_calls provided")
+
+        actual_calls = [
+            tc.name
+            for step in case.agent_trace
+            for tc in step.tool_calls
+        ]
+        expected = case.expected_tool_calls
+
+        if self.require_order:
+            # Ordered match
+            matches = sum(1 for a, e in zip(actual_calls, expected) if a == e)
+            score = matches / len(expected)
+            missing = [e for e in expected if e not in actual_calls]
+            unexpected = [a for a in actual_calls if a not in expected]
+        else:
+            # Unordered: fraction of expected tools that were called
+            called_set = set(actual_calls)
+            expected_set = set(expected)
+            matched = called_set & expected_set
+            score = len(matched) / len(expected_set)
+            missing = list(expected_set - called_set)
+            unexpected = list(called_set - expected_set)
+
+        reasons = [f"Called: {actual_calls}", f"Expected: {expected}"]
+        if missing:
+            reasons.append(f"Missing tools: {missing}")
+        if unexpected:
+            reasons.append(f"Unexpected tools: {unexpected}")
+
+        return self._result(score, "\n".join(reasons))
+
+
+class ToolArgumentAccuracy(Evaluator):
+    """
+    Evaluates whether tool arguments were correct and well-formed.
+    Uses LLM judge to assess argument quality since exact matching is too rigid.
+    Requires case.agent_trace.
+    """
+    name = "tool_argument_accuracy"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.agent_trace:
+            return self._result(0.0, "No agent_trace provided")
+
+        all_tool_calls = [tc for step in case.agent_trace for tc in step.tool_calls]
+        if not all_tool_calls:
+            return self._result(1.0, "No tool calls in trace")
+
+        results, reasons = [], []
+        for tc in all_tool_calls[:8]:
+            args_str = json.dumps(tc.arguments, indent=2) if tc.arguments else "{}"
+            prompt = (
+                f"Task: {case.input}\n\n"
+                f"Tool called: {tc.name}\n"
+                f"Arguments provided:\n{args_str}\n\n"
+                f"Are these arguments appropriate and well-formed for the tool '{tc.name}' given the task?"
+                f"\nAnswer \"Yes\" or \"No\"."
+            )
+            try:
+                answer = _judge_call(prompt, max_tokens=10)
+                good = _parse_yes_no(answer)
+                results.append(good)
+                reasons.append(f"{'✓' if good else '✗'} {tc.name}({args_str[:60]})")
+            except Exception as e:
+                results.append(False)
+                reasons.append(f"✗ {tc.name} (error: {e})")
+
+        score = sum(results) / len(results) if results else 0.0
+        return self._result(score, "\n".join(reasons))
+
+
+class PlanQuality(Evaluator):
+    """
+    Evaluates whether the agent's plan is logical, complete, and efficient.
+    Looks at the sequence of steps and tool calls as a whole.
+    Requires case.agent_trace.
+    """
+    name = "plan_quality"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.agent_trace:
+            return self._result(0.0, "No agent_trace provided")
+
+        trace = _trace_str(case.agent_trace)
+        ctx = f"Task: {case.input}\n\nAgent execution trace:\n{trace}\n\nFinal output: {output}"
+        questions = [
+            ("Does the agent's plan address all aspects of the task?", True),
+            ("Are the steps in the agent's plan in a logical order?", True),
+            ("Does the agent avoid redundant or unnecessary steps?", True),
+            ("Does each step in the plan follow logically from the previous one?", True),
+            ("Would an expert consider this plan efficient for the task?", True),
+        ]
+        score, reasons = _qag_eval(questions, ctx)
+        return self._result(score, "\n".join(reasons))
+
+
+class TaskCompletion(Evaluator):
+    """
+    Evaluates whether the agent successfully completed the given task.
+    Assesses the final output against the task goal — not just the process.
+    """
+    name = "task_completion"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        trace_str = ""
+        if case.agent_trace:
+            trace_str = f"\n\nAgent trace summary:\n{_trace_str(case.agent_trace)}"
+
+        ctx = f"Task: {case.input}{trace_str}\n\nFinal output: {output}"
+        questions = [
+            ("Does the final output successfully complete the task?", True),
+            ("Does the final output address all requirements of the task?", True),
+            ("Is the final output a complete response (not partial or cut off)?", True),
+            ("Did the agent fail to complete the task or produce an error?", False),
+        ]
+        score, reasons = _qag_eval(questions, ctx)
+        return self._result(score, "\n".join(reasons))
+
+
+class StepFaithfulness(Evaluator):
+    """
+    Evaluates whether each agent step faithfully follows from the task and prior steps.
+    Detects hallucinated reasoning or steps that contradict the task.
+    Requires case.agent_trace.
+    """
+    name = "step_faithfulness"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.agent_trace:
+            return self._result(0.0, "No agent_trace provided")
+
+        results, reasons = [], []
+        for i, step in enumerate(case.agent_trace[:8], 1):
+            prior = _trace_str(case.agent_trace[:i-1]) if i > 1 else "(no prior steps)"
+            step_str = _trace_str([step])
+            prompt = (
+                f"Task: {case.input}\n\n"
+                f"Prior steps:\n{prior}\n\n"
+                f"Current step {i}:\n{step_str}\n\n"
+                f"Does this step follow logically from the task and prior steps, "
+                f"without introducing contradictions or hallucinated information?"
+                f"\nAnswer \"Yes\" or \"No\"."
+            )
+            try:
+                answer = _judge_call(prompt, max_tokens=10)
+                faithful = _parse_yes_no(answer)
+                results.append(faithful)
+                thought_preview = step.thought[:60] if step.thought else "(no thought)"
+                reasons.append(f"{'✓' if faithful else '✗'} Step {i}: {thought_preview}")
+            except Exception as e:
+                results.append(False)
+                reasons.append(f"✗ Step {i} (error: {e})")
+
+        score = sum(results) / len(results) if results else 0.0
+        return self._result(score, f"{sum(results)}/{len(results)} steps faithful\n" + "\n".join(reasons))

multivon_eval/evaluators/base.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from ..case import EvalCase
+from ..result import EvalResult
+
+
+class Evaluator(ABC):
+    """Base class for all evaluators."""
+
+    name: str = "evaluator"
+    threshold: float = 0.5
+
+    def __init__(self, threshold: float = 0.5):
+        self.threshold = threshold
+
+    @abstractmethod
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        ...
+
+    def _result(self, score: float, reason: str = "", **metadata) -> EvalResult:
+        return EvalResult(
+            evaluator=self.name,
+            score=score,
+            passed=score >= self.threshold,
+            reason=reason,
+            metadata=metadata,
+        )

multivon_eval/evaluators/conversation.py

@@ -0,0 +1,136 @@
+"""
+Conversation evaluators — evaluate multi-turn dialogue quality.
+
+These evaluators assess the quality of conversational AI systems
+over the full dialogue, not just a single response.
+"""
+from __future__ import annotations
+
+from .base import Evaluator
+from .llm_judge import _judge_call, _parse_yes_no, _qag_eval
+from ..case import EvalCase
+from ..result import EvalResult
+
+
+class ConversationRelevance(Evaluator):
+    """
+    Evaluates whether each assistant turn is relevant to the ongoing conversation.
+    Detects responses that ignore prior context or change subject unexpectedly.
+    Requires case.conversation.
+    """
+    name = "conversation_relevance"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.conversation:
+            return self._result(0.0, "No conversation provided")
+
+        ctx = f"Conversation history:\n{case.conversation_str()}\n\nLatest response: {output}"
+        questions = [
+            ("Is the latest response relevant to the conversation history?", True),
+            ("Does the response address what the user was asking or discussing?", True),
+            ("Does the response ignore important context from earlier in the conversation?", False),
+            ("Does the response follow naturally from the preceding turns?", True),
+        ]
+        score, reasons = _qag_eval(questions, ctx)
+        return self._result(score, "\n".join(reasons))
+
+
+class KnowledgeRetention(Evaluator):
+    """
+    Evaluates whether the model retains and correctly uses facts established
+    earlier in the conversation.
+
+    Requires case.conversation — checks that the final response (output)
+    is consistent with facts introduced in earlier turns.
+    """
+    name = "knowledge_retention"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.conversation:
+            return self._result(0.0, "No conversation provided")
+
+        # Extract facts from user turns earlier in conversation
+        user_turns = [m["content"] for m in case.conversation if m["role"] == "user"]
+        if not user_turns:
+            return self._result(1.0, "No user turns to retain facts from")
+
+        ctx = (
+            f"Conversation history:\n{case.conversation_str()}\n\n"
+            f"Latest response:\n{output}"
+        )
+        questions = [
+            ("Does the response correctly recall facts mentioned by the user earlier in the conversation?", True),
+            ("Does the response contradict information the user provided in a prior turn?", False),
+            ("Does the response show awareness of the user's preferences or context established earlier?", True),
+        ]
+        score, reasons = _qag_eval(questions, ctx)
+        return self._result(score, "\n".join(reasons))
+
+
+class ConversationCompleteness(Evaluator):
+    """
+    Evaluates whether the assistant fully resolved the user's goals
+    over the course of the conversation.
+
+    Assesses the final output as the culmination of a multi-turn dialogue.
+    Requires case.conversation.
+    """
+    name = "conversation_completeness"
+
+    def __init__(self, threshold: float = 0.7):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.conversation:
+            return self._result(0.0, "No conversation provided")
+
+        # Infer the user's original goal from first user turn
+        first_user = next(
+            (m["content"] for m in case.conversation if m["role"] == "user"), case.input
+        )
+        ctx = (
+            f"Original user goal: {first_user}\n\n"
+            f"Full conversation:\n{case.conversation_str()}\n\n"
+            f"Final response:\n{output}"
+        )
+        questions = [
+            ("By the end of the conversation, has the user's original goal been fully addressed?", True),
+            ("Has the assistant left important questions from the user unanswered?", False),
+            ("Does the final response bring the conversation to a satisfying resolution?", True),
+            ("Would the user need to ask follow-up questions to get what they originally wanted?", False),
+        ]
+        score, reasons = _qag_eval(questions, ctx)
+        return self._result(score, "\n".join(reasons))
+
+
+class TurnConsistency(Evaluator):
+    """
+    Checks that the assistant doesn't contradict itself across turns.
+    Requires case.conversation.
+    """
+    name = "turn_consistency"
+
+    def __init__(self, threshold: float = 0.8):
+        super().__init__(threshold)
+
+    def evaluate(self, case: EvalCase, output: str) -> EvalResult:
+        if not case.conversation:
+            return self._result(0.0, "No conversation provided")
+
+        ctx = (
+            f"Conversation:\n{case.conversation_str()}\n\n"
+            f"Latest response:\n{output}"
+        )
+        questions = [
+            ("Is the latest response consistent with all prior assistant responses in the conversation?", True),
+            ("Does the assistant contradict something it said in a previous turn?", False),
+            ("Does the assistant maintain a consistent persona and tone throughout?", True),
+        ]
+        score, reasons = _qag_eval(questions, ctx)
+        return self._result(score, "\n".join(reasons))