eval-ai-library 0.1.0__py3-none-any.whl

eval_lib/agent_metrics/task_success_metric/task_success_rate.py

@@ -0,0 +1,347 @@
+# task_success_rate.py
+"""
+Task Success Rate Metric: Evaluates whether the AI assistant successfully helped
+the user achieve their goal in a conversation.
+
+Score calculation: Softmax aggregation of success criteria verdicts
+"""
+import json
+from typing import List, Dict, Any, Tuple
+from eval_lib.testcases_schema import ConversationalEvalTestCase
+from eval_lib.metric_pattern import ConversationalMetricPattern
+from eval_lib.llm_client import chat_complete
+from eval_lib.utils import score_agg, extract_json_block
+
+
+# Verdict weights for task completion levels
+VERDICT_WEIGHTS = {
+    "fully": 1.0,      # Criterion completely satisfied
+    "mostly": 0.9,     # Criterion largely satisfied with minor gaps
+    "partial": 0.7,    # Criterion partially satisfied
+    "minor": 0.3,      # Criterion minimally addressed
+    "none": 0.0        # Criterion not satisfied at all
+}
+
+# Configuration constants
+MAX_CRITERIA = 10
+LINK_CRITERION = "The user got the link to the requested resource."
+
+
+class TaskSuccessRateMetric(ConversationalMetricPattern):
+    """
+    Evaluates whether an AI assistant successfully helped the user complete
+    their intended task across a multi-turn conversation.
+    """
+
+    name = "taskSuccessRateMetric"
+    template_cls = None
+
+    def __init__(
+        self,
+        model: str,
+        threshold: float = 0.7,
+        temperature: float = 1.1,
+    ):
+        """
+        Initialize Task Success Rate metric.
+
+        Args:
+            model: LLM model name
+            threshold: Success threshold (0.0-1.0)
+            temperature: Score aggregation temperature for softmax
+        """
+        super().__init__(model=model, threshold=threshold)
+        self.temperature = temperature
+
+    # ==================== HELPER METHODS ====================
+
+    @staticmethod
+    def _render_dialogue(turns) -> str:
+        """Convert conversation turns into readable format"""
+        return "\n".join(
+            f"{i+1}. User: {t.input}\n   Assistant: {t.actual_output}"
+            for i, t in enumerate(turns)
+        )
+
+    @staticmethod
+    def _prompt_label_help() -> str:
+        """Explanation of task success verdict levels"""
+        return """Rate task success criteria satisfaction (worst → best):
+
+none    – criterion not satisfied at all
+minor   – criterion minimally addressed
+partial – criterion partially satisfied
+mostly  – criterion largely satisfied with minor gaps
+fully   – criterion completely satisfied"""
+
+    @staticmethod
+    def _prompt_criteria_few_shot() -> str:
+        """Few-shot examples for criteria generation"""
+        return """Example 1:
+User goal: Order a pizza online
+Criteria: [
+  "The assistant provided available pizza options.",
+  "The user received an order confirmation number."
+]
+
+Example 2:
+User goal: Reset an email password
+Criteria: [
+  "The assistant gave a working password-reset link.",
+  "The user confirmed they could log in."
+]"""
+
+    # ==================== CORE EVALUATION STEPS ====================
+
+    async def _infer_user_goal(self, dialogue: str) -> Tuple[str, float]:
+        """
+        Infer the user's primary goal from the conversation.
+
+        Args:
+            dialogue: Formatted conversation text
+
+        Returns:
+            Tuple of (user_goal_description, llm_cost)
+        """
+        prompt = (
+            "You will be shown an ENTIRE conversation between a user and an assistant.\n"
+            "Write ONE concise sentence describing the user's PRIMARY GOAL in this conversation.\n\n"
+            f"CONVERSATION:\n{dialogue}\n\n"
+            "User goal:"
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        return text.strip(), cost or 0.0
+
+    async def _generate_success_criteria(self, goal: str) -> Tuple[List[str], float]:
+        """
+        Generate concrete success criteria for the user's goal.
+
+        Args:
+            goal: The inferred user goal
+
+        Returns:
+            Tuple of (criteria_list, llm_cost)
+        """
+        prompt = (
+            f"{self._prompt_criteria_few_shot()}\n\n"
+            f"Now do the same for the next case.\n\n"
+            f"User goal: {goal}\n\n"
+            f"List up to {MAX_CRITERIA} concrete SUCCESS CRITERIA that could realistically be satisfied "
+            f"within a brief chat of 2–5 turns. "
+            "Then **add** this exact sentence: "
+            f"\"{LINK_CRITERION}\"\n\n"
+            "Each criterion must be a short, observable statement.\n"
+            "Return only a JSON array of strings."
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        try:
+            raw_json = extract_json_block(text)
+            criteria = json.loads(raw_json)
+
+            if not isinstance(criteria, list):
+                raise ValueError("Expected JSON array of criteria")
+
+            # Ensure LINK_CRITERION is included
+            if LINK_CRITERION not in criteria:
+                criteria.append(LINK_CRITERION)
+
+            # Keep LINK_CRITERION first and limit to MAX_CRITERIA
+            if len(criteria) > MAX_CRITERIA:
+                criteria = (
+                    [LINK_CRITERION] +
+                    [c for c in criteria if c != LINK_CRITERION][:MAX_CRITERIA - 1]
+                )
+
+            # Truncate to MAX_CRITERIA
+            criteria = criteria[:MAX_CRITERIA]
+
+            return criteria, cost or 0.0
+
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to parse success criteria: {e}\n{text}")
+
+    async def _generate_verdicts(
+        self,
+        goal: str,
+        criteria: List[str],
+        dialogue: str
+    ) -> Tuple[List[Dict[str, str]], float, float]:
+        """
+        Generate verdicts for each success criterion.
+
+        Args:
+            goal: The user's goal
+            criteria: List of success criteria
+            dialogue: Formatted conversation text
+
+        Returns:
+            Tuple of (verdicts_list, aggregated_score, llm_cost)
+        """
+        prompt = (
+            f"{self._prompt_label_help()}\n\n"
+            f"USER GOAL: {goal}\n\n"
+            f"FULL DIALOGUE:\n{dialogue}\n\n"
+            f"SUCCESS CRITERIA (as JSON array):\n{json.dumps(criteria, ensure_ascii=False)}\n\n"
+            "For **each** criterion, decide how well it is satisfied at the END of the dialogue.\n"
+            "Use exactly one of: fully, mostly, partial, minor, none.\n\n"
+            "Return JSON array with **exactly the same length and order** as the criteria list:\n"
+            "[{\"verdict\":\"fully|mostly|partial|minor|none\",\"reason\":\"<one sentence>\"}, …]\n\n"
+            "No extra text."
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        try:
+            raw_json = extract_json_block(text)
+            verdicts = json.loads(raw_json)
+
+            if not isinstance(verdicts, list):
+                raise ValueError("Expected JSON array of verdicts")
+
+            # Ensure verdicts match criteria length
+            if len(verdicts) != len(criteria):
+                # Pad or truncate to match
+                if len(verdicts) < len(criteria):
+                    verdicts.extend(
+                        [{"verdict": "none", "reason": "Missing evaluation"}] * (len(criteria) - len(verdicts)))
+                else:
+                    verdicts = verdicts[:len(criteria)]
+
+            # Calculate aggregated score from verdicts
+            weights = [VERDICT_WEIGHTS.get(
+                v.get("verdict", "none"), 0.0) for v in verdicts]
+            score = round(score_agg(weights, temperature=self.temperature), 4)
+
+            return verdicts, score, cost or 0.0
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to parse verdicts: {e}\n{text}")
+
+    async def _summarize_verdicts(
+        self,
+        verdicts: List[Dict[str, str]]
+    ) -> Tuple[str, float]:
+        """
+        Generate concise summary of task success assessment.
+
+        Args:
+            verdicts: List of verdict objects with reasons
+
+        Returns:
+            Tuple of (summary_text, llm_cost)
+        """
+        # Take up to 6 most relevant verdicts for summary
+        bullets = "\n".join(f"- {v['reason']}" for v in verdicts[:6])
+
+        prompt = (
+            "Write a concise (max 2 sentences) overall assessment of task success, "
+            "based on these observations:\n\n"
+            f"{bullets}\n\n"
+            "Summary:"
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        return text.strip(), cost or 0.0
+
+    # ==================== MAIN EVALUATION ====================
+
+    async def evaluate(self, test_case: ConversationalEvalTestCase) -> Dict[str, Any]:
+        """
+        Evaluate task success rate across conversation turns.
+
+        Steps:
+        1. Format dialogue into readable text
+        2. Infer user's primary goal from conversation
+        3. Generate concrete success criteria for the goal
+        4. Generate verdicts for each criterion (fully/mostly/partial/minor/none)
+        5. Aggregate verdicts into final score using softmax
+        6. Generate summary explanation
+        7. Build comprehensive evaluation log
+
+        Args:
+            test_case: Conversational test case with multiple turns
+
+        Returns:
+            Evaluation results with score, success, reason, cost, and detailed log
+        """
+        total_cost = 0.0
+
+        # Step 1: Format dialogue
+        dialogue_text = self._render_dialogue(test_case.turns)
+
+        # Step 2: Infer user goal
+        user_goal, cost = await self._infer_user_goal(dialogue_text)
+        total_cost += cost
+
+        # Step 3: Generate success criteria
+        success_criteria, cost = await self._generate_success_criteria(user_goal)
+        total_cost += cost
+
+        # Step 4: Generate verdicts for each criterion
+        verdicts, verdict_score, cost = await self._generate_verdicts(
+            user_goal,
+            success_criteria,
+            dialogue_text
+        )
+        total_cost += cost
+
+        # Step 5: Generate summary explanation
+        summary, cost = await self._summarize_verdicts(verdicts)
+        total_cost += cost
+
+        # Step 6: Determine success
+        final_score = verdict_score
+        success = final_score >= self.threshold
+
+        # Step 7: Build evaluation log
+        evaluation_log = {
+            "dialogue": dialogue_text,
+            "comment_dialogue": "Full conversation text used for task success evaluation.",
+            "number_of_turns": len(test_case.turns),
+            "comment_number_of_turns": "Total conversation turns analyzed.",
+            "user_goal": user_goal,
+            "comment_user_goal": "LLM-inferred primary goal the user wanted to achieve.",
+            "success_criteria": success_criteria,
+            "comment_success_criteria": f"Auto-generated checklist of {len(success_criteria)} observable criteria for task completion.",
+            "verdicts": verdicts,
+            "comment_verdicts": "LLM-generated verdicts assessing each criterion (fully/mostly/partial/minor/none).",
+            "verdict_weights": {i: VERDICT_WEIGHTS.get(v["verdict"], 0.0) for i, v in enumerate(verdicts)},
+            "comment_verdict_weights": "Numeric weights assigned to each verdict for score calculation.",
+            "final_score": final_score,
+            "comment_final_score": f"Softmax aggregation of verdict weights (temperature={self.temperature}).",
+            "threshold": self.threshold,
+            "success": success,
+            "comment_success": "Whether the task success score meets the required threshold.",
+            "final_reason": summary,
+            "comment_reasoning": "Concise explanation of the overall task completion assessment."
+        }
+
+        return {
+            "score": final_score,
+            "success": success,
+            "reason": summary,
+            "evaluation_cost": round(total_cost, 6),
+            "evaluation_log": evaluation_log
+        }

eval_lib/agent_metrics/tools_correctness_metric/tool_correctness.py

@@ -0,0 +1,106 @@
+'''
+Tool Correctness Metric: Evaluates whether the correct tools were called
+during the execution of an AI agent.
+Score calculation: Proportion of expected tools correctly called
+'''
+
+from typing import Dict, Any, List
+from eval_lib.metric_pattern import MetricPattern
+from eval_lib.testcases_schema import EvalTestCase
+
+
+class ToolCorrectnessMetric(MetricPattern):
+    name = "toolCorrectnessMetric"
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        evaluation_params: List[str] = [],
+        should_exact_match: bool = False,
+        should_consider_ordering: bool = False
+    ):
+        super().__init__(model=None, threshold=threshold)
+        self.evaluation_params = evaluation_params
+        self.should_exact_match = should_exact_match
+        self.should_consider_ordering = should_consider_ordering
+
+    async def evaluate(self, test_case: EvalTestCase) -> Dict[str, Any]:
+        self.tools_called = test_case.tools_called or []
+        self.expected_tools = test_case.expected_tools or []
+
+        score = self.calculate_score()
+        reason = self.generate_reason()
+
+        return {
+            "score": score,
+            "success": score >= self.threshold,
+            "reason": reason,
+            "evaluation_cost": 0.0  # No LLM cost for this metric
+        }
+
+    def generate_reason(self) -> str:
+        called_names = self.tools_called
+        expected_names = self.expected_tools
+
+        if self.should_exact_match:
+            if self.calculate_exact_match_score() == 1.0:
+                return f"Exact match: all expected tools {expected_names} were called exactly."
+            else:
+                return f"Mismatch: expected {expected_names}, called {called_names}."
+        elif self.should_consider_ordering:
+            lcs, weighted = self.compute_weighted_lcs()
+            if weighted == len(self.expected_tools):
+                return "Correct tool usage and order."
+            else:
+                return f"Incomplete or unordered: expected {expected_names}, got {called_names}."
+        else:
+            used_expected = set(self.tools_called) & set(self.expected_tools)
+            missing = set(self.expected_tools) - used_expected
+            if not missing:
+                return f"All expected tools {expected_names} were called."
+            else:
+                return f"Missing tools {list(missing)}. Expected {expected_names}, got {called_names}."
+
+    def calculate_score(self) -> float:
+        if self.should_exact_match:
+            return self.calculate_exact_match_score()
+        elif self.should_consider_ordering:
+            _, score = self.compute_weighted_lcs()
+            return score / len(self.expected_tools) if self.expected_tools else 0.0
+        else:
+            return self.calculate_non_exact_match_score()
+
+    def calculate_exact_match_score(self) -> float:
+        if len(self.tools_called) != len(self.expected_tools):
+            return 0.0
+        for i in range(len(self.tools_called)):
+            if self.tools_called[i] != self.expected_tools[i]:
+                return 0.0
+        return 1.0
+
+    def calculate_non_exact_match_score(self) -> float:
+        match_count = 0
+        used = set()
+        for expected in self.expected_tools:
+            for i, called in enumerate(self.tools_called):
+                if i in used:
+                    continue
+                if expected == called:
+                    match_count += 1
+                    used.add(i)
+                    break
+        return match_count / len(self.expected_tools) if self.expected_tools else 0.0
+
+    def compute_weighted_lcs(self):
+        m, n = len(self.expected_tools), len(self.tools_called)
+        dp = [[0.0] * (n + 1) for _ in range(m + 1)]
+
+        for i in range(1, m + 1):
+            for j in range(1, n + 1):
+                if self.expected_tools[i - 1] == self.tools_called[j - 1]:
+                    dp[i][j] = dp[i - 1][j - 1] + 1
+                else:
+                    dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
+
+        score = dp[m][n]
+        return [], score

eval_lib/datagenerator/datagenerator.py

@@ -0,0 +1,230 @@
+from typing import List
+from eval_lib.llm_client import chat_complete
+from .document_loader import load_documents, chunk_documents
+import math
+from eval_lib.llm_client import get_embeddings
+import numpy as np
+from .prompts import dataset_generation_prompt, dataset_generation_from_scratch_prompt
+from eval_lib.utils import extract_json_block
+import asyncio
+import random
+import json
+
+
+async def retry_async(fn, *args, retries=4, base_delay=0.6, max_delay=6.0,
+                      retriable_statuses=(429, 500, 502, 503, 504),
+                      **kwargs):
+    """
+    fn — корутина, которая может бросить исключение вида:
+    - HTTPException-like с .status_code
+    - Exception с текстом, где встречается 'Service Unavailable' и т.п.
+    """
+    attempt = 0
+    while True:
+        try:
+            return await fn(*args, **kwargs)
+        except Exception as e:
+            attempt += 1
+            status = getattr(e, "status_code", None)
+            msg = str(e).lower()
+
+            retriable = (status in retriable_statuses) or any(
+                s in msg for s in ["service unavailable", "temporarily unavailable",
+                                   "gateway timeout", "bad gateway", "timeout"])
+            if attempt > retries or not retriable:
+                raise
+
+            # экспоненциальный бэкофф + джиттер
+            delay = min(max_delay, base_delay * (2 ** (attempt - 1)))
+            delay += random.uniform(0, 0.4)
+            await asyncio.sleep(delay)
+
+
+class DatasetGenerator:
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        input_format: str,
+        expected_output_format: str,
+        agent_description: str,
+        test_types: List[str],
+        question_length: str = "mixed",
+        question_openness: str = "mixed",
+        chunk_size: int = 1024,
+        chunk_overlap: int = 100,
+        temperature: float = 0.3,
+        max_rows: int = 10,
+        trap_density: float = 0.1,
+        language: str = "en",
+        max_chunks: int = 30,
+        relevance_margin: float = 1.5,
+        embedding_model: str = "openai:text-embedding-3-small",
+    ):
+        self.model = model
+        self.input_format = input_format
+        self.expected_output_format = expected_output_format
+        self.agent_description = agent_description
+        self.test_types = test_types
+        self.question_length = question_length
+        self.question_openness = question_openness
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.temperature = temperature
+        self.max_rows = max_rows
+        self.trap_density = trap_density
+        self.language = language
+        self.max_chunks = max_chunks
+        self.relevance_margin = relevance_margin
+        self.embedding_model = embedding_model
+
+    async def generate_from_scratch(self) -> List[dict]:
+        prompt = dataset_generation_from_scratch_prompt(
+            max_rows=self.max_rows,
+            agent_description=self.agent_description,
+            input_format=self.input_format,
+            expected_output_format=self.expected_output_format,
+            test_types=self.test_types,
+            question_length=self.question_length,
+            question_openness=self.question_openness,
+            trap_density=self.trap_density,
+            language=self.language
+        )
+
+        raw, _ = await chat_complete(
+            llm=self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.temperature,
+        )
+
+        try:
+            raw_json = extract_json_block(raw)
+            data = json.loads(raw_json)
+            assert isinstance(data, list), "not a JSON array"
+            return data
+        except Exception as exc:
+            raise RuntimeError(f"Failed to parse dataset:\n{exc}\n\n{raw}")
+
+    async def generate_from_documents(self, file_paths: List[str]) -> List[dict]:
+
+        docs = load_documents(file_paths)
+        doc_chunks = chunk_documents(docs,
+                                     chunk_size=self.chunk_size,
+                                     chunk_overlap=self.chunk_overlap)
+
+        chunks_text = [d.page_content for d in doc_chunks]
+        if not chunks_text:
+            raise ValueError("No text extracted from documents.")
+
+        ranked_chunks = await self._rank_chunks_by_relevance(chunks_text)
+
+        total_chunks = len(ranked_chunks)
+        rows_per_chunk = max(1, math.ceil(self.max_rows / total_chunks))
+
+        needed_chunks = math.ceil(self.max_rows / rows_per_chunk)
+        top_k = min(int(needed_chunks * self.relevance_margin),
+                    self.max_chunks)
+        selected_chunks = ranked_chunks[:top_k]
+
+        dataset: list[dict] = []
+
+        MAX_PROMPT_CHARS = 24_000
+
+        for chunk in selected_chunks:
+
+            safe_chunk = chunk if len(
+                chunk) <= MAX_PROMPT_CHARS else chunk[:MAX_PROMPT_CHARS]
+
+            prompt = dataset_generation_prompt(
+                chunk=safe_chunk,
+                rows_per_chunk=rows_per_chunk,
+                agent_description=self.agent_description,
+                input_format=self.input_format,
+                expected_output_format=self.expected_output_format,
+                test_types=self.test_types,
+                question_length=self.question_length,
+                question_openness=self.question_openness,
+                trap_density=self.trap_density,
+                language=self.language
+            )
+
+            raw, _ = await retry_async(
+                chat_complete,
+                llm=self.model,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=self.temperature,
+            )
+
+            try:
+                chunk_data = json.loads(extract_json_block(raw))
+                assert isinstance(chunk_data, list)
+                dataset.extend(chunk_data)
+            except Exception as exc:
+                raise RuntimeError(f"Chunk parsing error:\n{exc}\n\n{raw}")
+
+            if len(dataset) >= self.max_rows:
+                break
+
+        return dataset[: self.max_rows]
+
+    async def _rank_chunks_by_relevance(self, chunks: list[str]) -> list[str]:
+        """
+        Count token similarity between chunks and query.
+
+        """
+        # estimate tokens
+        def approx_tokens(s: str) -> int:
+            return max(1, len(s) // 4)
+
+        # restrict length of each chunk for embedding (e.g., to ~8k tokens)
+        MAX_EMBED_TOKENS_PER_INPUT = 8000
+        MAX_EMBED_CHARS_PER_INPUT = MAX_EMBED_TOKENS_PER_INPUT * 4
+
+        truncated_chunks = [
+            c if len(
+                c) <= MAX_EMBED_CHARS_PER_INPUT else c[:MAX_EMBED_CHARS_PER_INPUT]
+            for c in chunks
+        ]
+
+        # limit tokens per request
+        TOKEN_BUDGET_PER_REQUEST = 280_000
+
+        # divide into batches by total tokens
+        batches: list[list[str]] = []
+        cur: list[str] = []
+        cur_tokens = 0
+        for c in truncated_chunks:
+            t = approx_tokens(c)
+            if cur and (cur_tokens + t) > TOKEN_BUDGET_PER_REQUEST:
+                batches.append(cur)
+                cur = [c]
+                cur_tokens = t
+            else:
+                cur.append(c)
+                cur_tokens += t
+        if cur:
+            batches.append(cur)
+
+        # embedding for query
+        query = self.agent_description + " " + " ".join(self.test_types)
+        q_vec, _ = await retry_async(get_embeddings, model=self.embedding_model, texts=[query])
+        q_vec = q_vec[0]
+
+        # go through batches, accumulating embeddings
+        all_vecs = []
+        for batch in batches:
+            vecs, _ = await retry_async(get_embeddings, model=self.embedding_model, texts=batch)
+            all_vecs.extend(vecs)
+
+        import numpy as np
+        q_norm = np.linalg.norm(q_vec) + 1e-7
+        sims = [
+            float(np.dot(q_vec, v) / (q_norm * (np.linalg.norm(v) + 1e-7)))
+            for v in all_vecs
+        ]
+
+        # sort
+        ranked = [c for _, c in sorted(
+            zip(sims, chunks), key=lambda x: x[0], reverse=True)]
+        return ranked