eval-ai-library 0.1.0__py3-none-any.whl

eval_lib/evaluate.py

@@ -0,0 +1,335 @@
+# evaluate.py
+"""
+Main evaluation functions with beautiful console progress tracking.
+"""
+from dataclasses import asdict
+import json
+import time
+from typing import List, Tuple, Dict, Any
+from eval_lib.testcases_schema import EvalTestCase, ConversationalEvalTestCase
+from eval_lib.metric_pattern import MetricPattern, ConversationalMetricPattern
+from eval_lib.evaluation_schema import TestCaseResult, MetricResult, ConversationalTestCaseResult
+
+
+# ANSI color codes
+class Colors:
+    HEADER = '\033[95m'
+    BLUE = '\033[94m'
+    CYAN = '\033[96m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    RED = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+    DIM = '\033[2m'
+
+
+def _print_header(title: str):
+    """Print formatted header"""
+    print(f"\n{Colors.BOLD}{Colors.HEADER}{'='*70}{Colors.ENDC}")
+    print(f"{Colors.BOLD}{Colors.HEADER}{title.center(70)}{Colors.ENDC}")
+    print(f"{Colors.BOLD}{Colors.HEADER}{'='*70}{Colors.ENDC}\n")
+
+
+def _print_progress(current: int, total: int, item_name: str):
+    """Print progress bar"""
+    percentage = (current / total) * 100
+    bar_length = 40
+    filled = int(bar_length * current / total)
+    bar = '█' * filled + '░' * (bar_length - filled)
+
+    print(
+        f"\r{Colors.CYAN}Progress: [{bar}] {percentage:.0f}% ({current}/{total}) - {item_name}{Colors.ENDC}", end='', flush=True)
+
+
+def _print_summary(results: List, total_cost: float, total_time: float, passed: int, total: int):
+    """Print evaluation summary"""
+    print(f"\n\n{Colors.BOLD}{Colors.GREEN}{'='*70}{Colors.ENDC}")
+    print(f"{Colors.BOLD}{Colors.GREEN}📋 EVALUATION SUMMARY{Colors.ENDC}")
+    print(f"{Colors.BOLD}{Colors.GREEN}{'='*70}{Colors.ENDC}")
+
+    success_rate = (passed / total * 100) if total > 0 else 0
+    status_color = Colors.GREEN if success_rate >= 80 else Colors.YELLOW if success_rate >= 50 else Colors.RED
+
+    print(f"\n{Colors.BOLD}Overall Results:{Colors.ENDC}")
+    print(f"  ✅ Passed: {Colors.GREEN}{passed}{Colors.ENDC} / {total}")
+    print(f"  ❌ Failed: {Colors.RED}{total - passed}{Colors.ENDC} / {total}")
+    print(f"  📊 Success Rate: {status_color}{success_rate:.1f}%{Colors.ENDC}")
+    print(f"\n{Colors.BOLD}Resource Usage:{Colors.ENDC}")
+    print(f"  💰 Total Cost: {Colors.YELLOW}${total_cost:.6f}{Colors.ENDC}")
+    print(f"  ⏱️  Total Time: {Colors.CYAN}{total_time:.2f}s{Colors.ENDC}")
+    print(
+        f"  📈 Avg Time per Test: {Colors.DIM}{(total_time/total if total > 0 else 0):.2f}s{Colors.ENDC}")
+
+    print(f"\n{Colors.BOLD}{Colors.GREEN}{'='*70}{Colors.ENDC}\n")
+
+
+async def evaluate(
+    test_cases: List[EvalTestCase],
+    metrics: List[MetricPattern],
+    verbose: bool = True
+) -> List[Tuple[None, List[TestCaseResult]]]:
+    """
+    Evaluate test cases with multiple metrics.
+
+    Args:
+        test_cases: List of test cases to evaluate
+        metrics: List of metrics to apply
+        verbose: Enable detailed logging (default: True)
+
+    Returns:
+        List of evaluation results
+    """
+    start_time = time.time()
+    results: List[Tuple[None, List[TestCaseResult]]] = []
+
+    total_cost = 0.0
+    total_passed = 0
+    total_tests = len(test_cases)
+
+    if verbose:
+        _print_header("🚀 STARTING EVALUATION")
+        print(f"{Colors.BOLD}Configuration:{Colors.ENDC}")
+        print(f"  📝 Test Cases: {Colors.CYAN}{total_tests}{Colors.ENDC}")
+        print(f"  📊 Metrics: {Colors.CYAN}{len(metrics)}{Colors.ENDC}")
+        print(
+            f"  🎯 Total Evaluations: {Colors.CYAN}{total_tests * len(metrics)}{Colors.ENDC}")
+
+        print(f"\n{Colors.BOLD}Metrics:{Colors.ENDC}")
+        for i, m in enumerate(metrics, 1):
+            print(
+                f"  {i}. {Colors.BLUE}{m.name}{Colors.ENDC} (threshold: {m.threshold})")
+
+    # Process each test case
+    for tc_idx, tc in enumerate(test_cases, 1):
+        if verbose:
+            print(f"\n{Colors.BOLD}{Colors.CYAN}{'─'*70}{Colors.ENDC}")
+            print(
+                f"{Colors.BOLD}{Colors.CYAN}📝 Test Case {tc_idx}/{total_tests}{Colors.ENDC}")
+            print(f"{Colors.BOLD}{Colors.CYAN}{'─'*70}{Colors.ENDC}")
+            print(
+                f"{Colors.DIM}Input: {tc.input[:80]}{'...' if len(tc.input) > 80 else ''}{Colors.ENDC}")
+
+        mdata = []
+        test_cost = 0.0
+
+        # Evaluate with each metric
+        for m_idx, m in enumerate(metrics, 1):
+            if verbose:
+                _print_progress(m_idx, len(metrics), m.name)
+
+            # Set verbose flag for metrics
+            original_verbose = getattr(m, 'verbose', True)
+            m.verbose = verbose
+
+            res = await m.evaluate(tc)
+
+            # Restore original verbose setting
+            m.verbose = original_verbose
+
+            # Gather results
+            cost = res.get("evaluation_cost", 0) or 0
+            test_cost += cost
+            total_cost += cost
+
+            mdata.append(MetricResult(
+                name=m.name,
+                score=res["score"],
+                threshold=m.threshold,
+                success=res["success"],
+                evaluation_cost=cost,
+                reason=res["reason"],
+                evaluation_model=m.model,
+                evaluation_log=res.get("evaluation_log", None)
+            ))
+
+        overall = all(d.success for d in mdata)
+        if overall:
+            total_passed += 1
+
+        if verbose:
+            print(f"\n{Colors.BOLD}Test Case Summary:{Colors.ENDC}")
+            tc_status_color = Colors.GREEN if overall else Colors.RED
+            tc_status_icon = "✅" if overall else "❌"
+            print(
+                f"  {tc_status_icon} Overall: {tc_status_color}{Colors.BOLD}{'PASSED' if overall else 'FAILED'}{Colors.ENDC}")
+            print(f"  💰 Cost: {Colors.YELLOW}${test_cost:.6f}{Colors.ENDC}")
+
+            # Show metric breakdown
+            print(f"\n  {Colors.BOLD}Metrics Breakdown:{Colors.ENDC}")
+            for md in mdata:
+                status = "✅" if md.success else "❌"
+                color = Colors.GREEN if md.success else Colors.RED
+                print(
+                    f"    {status} {md.name}: {color}{md.score:.2f}{Colors.ENDC}")
+
+        results.append((None, [TestCaseResult(
+            input=tc.input,
+            actual_output=tc.actual_output,
+            expected_output=tc.expected_output,
+            retrieval_context=tc.retrieval_context,
+            tools_called=tc.tools_called,
+            expected_tools=tc.expected_tools,
+            success=overall,
+            metrics_data=mdata
+        )]))
+
+    # Calculate total time
+    total_time = time.time() - start_time
+
+    # Print summary
+    if verbose:
+        _print_summary(results, total_cost, total_time,
+                       total_passed, total_tests)
+
+    # Print detailed results if requested
+    if verbose:
+        print(f"{Colors.BOLD}Detailed Results:{Colors.ENDC}")
+        for idx, (meta, tc_list) in enumerate(results, 1):
+            print(f"\n{Colors.DIM}{'─'*70}{Colors.ENDC}")
+            print(f"{Colors.BOLD}Test Case {idx}:{Colors.ENDC}")
+            for tc in tc_list:
+                tc_dict = asdict(tc)
+                # Pretty print with indentation
+                print(json.dumps(tc_dict, indent=2, ensure_ascii=False))
+        print(f"{Colors.DIM}{'─'*70}{Colors.ENDC}\n")
+
+    return results
+
+
+async def evaluate_conversations(
+    conv_cases: List[ConversationalEvalTestCase],
+    metrics: List[ConversationalMetricPattern],
+    verbose: bool = True
+) -> List[Tuple[None, List[ConversationalTestCaseResult]]]:
+    """
+    Evaluate conversational test cases with multiple metrics.
+
+    Args:
+        conv_cases: List of conversational test cases
+        metrics: List of conversational metrics
+        verbose: Enable detailed logging (default: True)
+
+    Returns:
+        List of evaluation results
+    """
+    start_time = time.time()
+    results: List[Tuple[None, List[ConversationalTestCaseResult]]] = []
+
+    total_cost = 0.0
+    total_passed = 0
+    total_conversations = len(conv_cases)
+
+    if verbose:
+        _print_header("🚀 STARTING CONVERSATIONAL EVALUATION")
+        print(f"{Colors.BOLD}Configuration:{Colors.ENDC}")
+        print(
+            f"  💬 Conversations: {Colors.CYAN}{total_conversations}{Colors.ENDC}")
+        print(f"  📊 Metrics: {Colors.CYAN}{len(metrics)}{Colors.ENDC}")
+        print(
+            f"  🎯 Total Evaluations: {Colors.CYAN}{total_conversations * len(metrics)}{Colors.ENDC}")
+
+        print(f"\n{Colors.BOLD}Metrics:{Colors.ENDC}")
+        for i, m in enumerate(metrics, 1):
+            print(
+                f"  {i}. {Colors.BLUE}{m.name}{Colors.ENDC} (threshold: {m.threshold})")
+
+    # Process each conversation
+    for conv_idx, dlg in enumerate(conv_cases, 1):
+        if verbose:
+            print(f"\n{Colors.BOLD}{Colors.CYAN}{'─'*70}{Colors.ENDC}")
+            print(
+                f"{Colors.BOLD}{Colors.CYAN}💬 Conversation {conv_idx}/{total_conversations}{Colors.ENDC}")
+            print(f"{Colors.BOLD}{Colors.CYAN}{'─'*70}{Colors.ENDC}")
+            print(f"{Colors.DIM}Turns: {len(dlg.turns)}{Colors.ENDC}")
+            if dlg.chatbot_role:
+                print(
+                    f"{Colors.DIM}Role: {dlg.chatbot_role[:60]}{'...' if len(dlg.chatbot_role) > 60 else ''}{Colors.ENDC}")
+
+        metric_rows: List[MetricResult] = []
+        conv_cost = 0.0
+
+        # Evaluate with each metric
+        for m_idx, m in enumerate(metrics, 1):
+            if verbose:
+                _print_progress(m_idx, len(metrics), m.name)
+
+            # Set verbose flag for metrics
+            original_verbose = getattr(m, 'verbose', True)
+            m.verbose = verbose
+
+            res: Dict[str, Any] = await m.evaluate(dlg)
+
+            # Restore original verbose setting
+            m.verbose = original_verbose
+
+            cost = res.get("evaluation_cost", 0) or 0
+            conv_cost += cost
+            total_cost += cost
+
+            metric_rows.append(
+                MetricResult(
+                    name=m.name,
+                    score=res["score"],
+                    threshold=m.threshold,
+                    success=res["success"],
+                    evaluation_cost=cost,
+                    reason=res.get("reason"),
+                    evaluation_model=m.model,
+                    evaluation_log=res.get("evaluation_log"),
+                )
+            )
+
+        overall_ok = all(r.success for r in metric_rows)
+        if overall_ok:
+            total_passed += 1
+
+        if verbose:
+            print(f"\n{Colors.BOLD}Conversation Summary:{Colors.ENDC}")
+            conv_status_color = Colors.GREEN if overall_ok else Colors.RED
+            conv_status_icon = "✅" if overall_ok else "❌"
+            print(
+                f"  {conv_status_icon} Overall: {conv_status_color}{Colors.BOLD}{'PASSED' if overall_ok else 'FAILED'}{Colors.ENDC}")
+            print(f"  💰 Cost: {Colors.YELLOW}${conv_cost:.6f}{Colors.ENDC}")
+
+            # Show metric breakdown
+            print(f"\n  {Colors.BOLD}Metrics Breakdown:{Colors.ENDC}")
+            for mr in metric_rows:
+                status = "✅" if mr.success else "❌"
+                color = Colors.GREEN if mr.success else Colors.RED
+                print(
+                    f"    {status} {mr.name}: {color}{mr.score:.2f}{Colors.ENDC}")
+
+        dialogue_raw = []
+        for turn in dlg.turns:
+            dialogue_raw.append({"role": "user", "content": turn.input})
+            dialogue_raw.append(
+                {"role": "assistant", "content": turn.actual_output})
+
+        conv_res = ConversationalTestCaseResult(
+            dialogue=dialogue_raw,
+            success=overall_ok,
+            metrics_data=metric_rows,
+        )
+        results.append((None, [conv_res]))
+
+    # Calculate total time
+    total_time = time.time() - start_time
+
+    # Print summary
+    if verbose:
+        _print_summary(results, total_cost, total_time,
+                       total_passed, total_conversations)
+
+    # Print detailed results if requested
+    if verbose:
+        print(f"{Colors.BOLD}Detailed Results:{Colors.ENDC}")
+        for idx, (_, conv_list) in enumerate(results, 1):
+            print(f"\n{Colors.DIM}{'─'*70}{Colors.ENDC}")
+            print(f"{Colors.BOLD}Conversation {idx}:{Colors.ENDC}")
+            for conv in conv_list:
+                print(json.dumps(asdict(conv), indent=2, ensure_ascii=False))
+        print(f"{Colors.DIM}{'─'*70}{Colors.ENDC}\n")
+
+    return results

eval_lib/evaluation_schema.py

@@ -0,0 +1,63 @@
+# evaluation_schema.py
+
+from dataclasses import dataclass
+from typing import List, Optional, Any, Dict
+
+
+@dataclass
+class MetricResult:
+    """
+    Result of one metric evaluation.
+    In pair with TestCaseResult used in save_results in the form of (MetricResult, None)
+    - name: str
+    - score: float
+    - threshold: float
+    - success: bool
+    - evaluation_cost: Optional[float]
+    - reason: Optional[str]
+    - evaluation_model: str
+    """
+    name: str
+    score: float
+    threshold: float
+    success: bool
+    evaluation_cost: Optional[float]
+    reason: Optional[str]
+    evaluation_model: str
+    # can be a dict or any other type depending on the metric
+    evaluation_log: Optional[Any] = None
+
+
+@dataclass
+class TestCaseResult:
+    """
+    Final result of one test case evaluation.
+    In pair with MetricResult used in save_results in the form of (None, [TestCaseResult])
+    - input:             str
+    - actual_output:     str    
+    - expected_output:   Optional[str]
+    - retrieval_context: Optional[List[str]]
+    - success:           bool               — General success of the test case
+    - metrics_data:      List[MetricResult] — List of metric results
+    """
+    input: str
+    actual_output: str
+    expected_output: Optional[str]
+    retrieval_context: Optional[List[str]]
+    success: bool
+    metrics_data: List[MetricResult]
+    tools_called: Optional[List[str]] = None
+    expected_tools: Optional[List[str]] = None
+
+
+@dataclass
+class ConversationalTestCaseResult:
+    """
+    Result of a conversational test case evaluation.
+    - turns: List[TestCaseResult] — List of individual test case results for each turn
+    - chatbot_role: Optional[str] — Role of the chatbot in the conversation
+    - name: Optional[str] — Name of the conversational test case
+    """
+    dialogue: List[Dict[str, str]]
+    success: bool
+    metrics_data: List[MetricResult]

eval_lib/llm_client.py

@@ -0,0 +1,286 @@
+# llm_client.py
+import openai
+import functools
+import anthropic
+from openai import AsyncAzureOpenAI
+from google import genai
+from google.genai.types import GenerateContentConfig
+import os
+from enum import Enum
+from dataclasses import dataclass
+from typing import Dict, Tuple, Optional
+from types import SimpleNamespace
+from .price import model_pricing
+
+
+class Provider(str, Enum):
+    OPENAI = "openai"
+    AZURE = "azure"
+    GOOGLE = "google"
+    OLLAMA = "ollama"
+    ANTHROPIC = "anthropic"
+
+
+@dataclass(frozen=True, slots=True)
+class LLMDescriptor:
+    """'openai:gpt-4o'  →  provider=openai, model='gpt-4o'"""
+    provider: Provider
+    model: str
+
+    @classmethod
+    def parse(cls, spec: str | Tuple[str, str] | "LLMDescriptor") -> "LLMDescriptor":
+        if isinstance(spec, LLMDescriptor):
+            return spec
+        if isinstance(spec, tuple):
+            provider, model = spec
+            return cls(Provider(provider), model)
+        try:
+            provider, model = spec.split(":", 1)
+        except ValueError:
+            return cls(Provider.OPENAI, spec)
+        return cls(Provider(provider), model)
+
+    def key(self) -> str:
+        """Return a unique key for the LLM descriptor."""
+        return f"{self.provider}:{self.model}"
+
+
+@functools.cache
+def _get_client(provider: Provider):
+    if provider == Provider.OPENAI:
+        return openai.AsyncOpenAI()
+
+    if provider == Provider.AZURE:
+        return AsyncAzureOpenAI(
+            api_key=os.getenv("AZURE_OPENAI_API_KEY"),
+            azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
+            api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+        )
+
+    if provider == Provider.GOOGLE:
+        return genai.Client(api_key=os.getenv("GOOGLE_API_KEY"))
+
+    if provider == Provider.OLLAMA:
+        return openai.AsyncOpenAI(
+            api_key=os.getenv("OLLAMA_API_KEY"),
+            base_url=os.getenv("OLLAMA_API_BASE_URL")
+        )
+
+    if provider == Provider.ANTHROPIC:
+        return anthropic.AsyncAnthropic(
+            api_key=os.getenv("ANTHROPIC_API_KEY"),
+        )
+
+    raise ValueError(f"Unsupported provider {provider}")
+
+
+async def _openai_chat_complete(
+    client,
+    llm: LLMDescriptor,
+    messages: list[dict[str, str]],
+    temperature: float,
+):
+    """
+    Обычный OpenAI.
+    """
+    response = await client.chat.completions.create(
+        model=llm.model,
+        messages=messages,
+        temperature=temperature,
+    )
+    text = response.choices[0].message.content.strip()
+    cost = _calculate_cost(llm, response.usage)
+    return text, cost
+
+
+async def _azure_chat_complete(
+    client,
+    llm: LLMDescriptor,
+    messages: list[dict[str, str]],
+    temperature: float,
+):
+
+    deployment_name = os.getenv("AZURE_OPENAI_DEPLOYMENT") or llm.model
+
+    response = await client.chat.completions.create(
+        model=deployment_name,
+        messages=messages,
+        temperature=temperature,
+    )
+    text = response.choices[0].message.content.strip()
+    cost = _calculate_cost(llm, response.usage)
+    return text, cost
+
+
+async def _google_chat_complete(
+    client,
+    llm: LLMDescriptor,
+    messages: list[dict[str, str]],
+    temperature: float,
+):
+    """
+    Google GenAI / Gemini 2.x 
+    """
+    prompt = "\n".join(m["content"] for m in messages)
+
+    response = await client.aio.models.generate_content(
+        model=llm.model,
+        contents=prompt,
+        config=GenerateContentConfig(temperature=temperature),
+    )
+
+    text = response.text.strip()
+
+    um = response.usage_metadata
+    usage = SimpleNamespace(
+        prompt_tokens=um.prompt_token_count,
+        completion_tokens=um.candidates_token_count,
+    )
+
+    cost = _calculate_cost(llm, usage)
+    return text, cost
+
+
+async def _ollama_chat_complete(
+    client,
+    llm: LLMDescriptor,
+    messages: list[dict[str, str]],
+    temperature: float,
+):
+    response = await client.chat.completions.create(
+        model=llm.model,
+        messages=messages,
+        temperature=temperature,
+    )
+    text = response.choices[0].message.content.strip()
+    cost = _calculate_cost(llm, response.usage)
+    return text, cost
+
+
+async def _anthropic_chat_complete(
+    client,
+    llm: LLMDescriptor,
+    messages: list[dict[str, str]],
+    temperature: float,
+):
+    """
+    Anthropic Claude chat completion.
+    """
+    response = await client.messages.create(
+        model=llm.model,
+        messages=messages,
+        temperature=temperature,
+        max_tokens=4096,  # Default max tokens for Claude
+    )
+    if isinstance(response.content, list):
+        text = "".join(
+            block.text for block in response.content if block.type == "text").strip()
+    else:
+        text = response.content.strip()
+
+    cost = _calculate_cost(llm, response.usage)
+    return text, cost
+
+
+_HELPERS = {
+    Provider.OPENAI: _openai_chat_complete,
+    Provider.AZURE: _azure_chat_complete,
+    Provider.GOOGLE: _google_chat_complete,
+    Provider.OLLAMA: _ollama_chat_complete,
+    Provider.ANTHROPIC: _anthropic_chat_complete,
+}
+
+
+async def chat_complete(
+    llm: str | tuple[str, str] | LLMDescriptor,
+    messages: list[dict[str, str]],
+    temperature: float = 0.0,
+):
+    llm = LLMDescriptor.parse(llm)
+    helper = _HELPERS.get(llm.provider)
+
+    if helper is None:
+        raise ValueError(f"Unsupported provider {llm.provider}")
+
+    client = _get_client(llm.provider)
+    return await helper(client, llm, messages, temperature)
+
+
+def _calculate_cost(llm: LLMDescriptor, usage) -> Optional[float]:
+    """
+    Calculate the cost of the LLM usage based on the model and usage data.
+    """
+    if llm.provider == Provider.OLLAMA:
+        return 0.0
+    if not usage:
+        return None
+
+    price = model_pricing.get(llm.model)
+    if not price:
+        return None
+
+    prompt = getattr(usage, "prompt_tokens",     0)
+    completion = getattr(usage, "completion_tokens", 0)
+
+    return round(
+        prompt * price["input"] / 1_000_000 +
+        completion * price["output"] / 1_000_000,
+        6
+    )
+
+
+async def get_embeddings(
+    model: str | tuple[str, str] | LLMDescriptor,
+    texts: list[str],
+) -> tuple[list[list[float]], Optional[float]]:
+    """
+    Get embeddings for a list of texts using OpenAI models.
+
+    Args:
+        model: Model specification (e.g., "openai:text-embedding-3-small")
+        texts: List of texts to embed
+
+    Returns:
+        Tuple of (embeddings_list, total_cost)
+    """
+    llm = LLMDescriptor.parse(model)
+
+    if llm.provider != Provider.OPENAI:
+        raise ValueError(
+            f"Only OpenAI embedding models are supported, got {llm.provider}")
+
+    client = _get_client(llm.provider)
+    return await _openai_get_embeddings(client, llm, texts)
+
+
+async def _openai_get_embeddings(
+    client,
+    llm: LLMDescriptor,
+    texts: list[str],
+) -> tuple[list[list[float]], Optional[float]]:
+    """OpenAI embeddings implementation."""
+    response = await client.embeddings.create(
+        model=llm.model,
+        input=texts,
+        encoding_format="float"
+    )
+
+    embeddings = [data.embedding for data in response.data]
+    cost = _calculate_embedding_cost(llm, response.usage)
+
+    return embeddings, cost
+
+
+def _calculate_embedding_cost(llm: LLMDescriptor, usage) -> Optional[float]:
+    """Calculate the cost of embedding usage for OpenAI models."""
+    if not usage:
+        return None
+
+    price = model_pricing.get(llm.model)
+    if not price:
+        return None
+
+    total_tokens = getattr(usage, 'total_tokens', 0)
+    input_price = price.get("input", 0)
+
+    return round(total_tokens * input_price / 1_000_000, 6)