eval-ai-library 0.1.0__py3-none-any.whl

eval_lib/metrics/custom_metric/custom_eval.py

@@ -0,0 +1,303 @@
+# custom_eval.py
+"""
+Custom Evaluation Metric with Chain-of-Thought and Probability-Weighted Scoring
+Uses advanced techniques from G-Eval for improved accuracy
+"""
+import json
+import re
+from typing import Dict, Any, List, Tuple
+from collections import Counter
+from eval_lib.metric_pattern import MetricPattern
+from eval_lib.testcases_schema import EvalTestCase
+from eval_lib.llm_client import chat_complete
+from eval_lib.utils import extract_json_block
+
+
+class CustomEvalMetric(MetricPattern):
+    name = "customEval"
+
+    def __init__(self, model: str, threshold: float, name: str, criteria: str):
+        super().__init__(model=model, threshold=threshold)
+        self.custom_name = name
+        self.criteria = criteria
+
+    # ==================== PROMPTS ====================
+
+    @staticmethod
+    def _prompt_generate_steps(criteria: str) -> str:
+        """Generate evaluation steps from criteria (Chain-of-Thought)"""
+        return f"""Given the evaluation criteria below, generate 3-5 detailed evaluation steps.
+
+Evaluation Criteria:
+{criteria}
+
+Generate steps that are:
+1. Specific and actionable
+2. Logically ordered
+3. Lead to assigning a score from 0.0 to 1.0
+
+**
+Return ONLY JSON:
+{{
+  "steps": ["Step 1: ...", "Step 2: ...", "Step 3: ..."]
+}}
+**
+
+JSON:"""
+
+    @staticmethod
+    def _prompt_evaluate(criteria: str, evaluation_steps: List[str], test_case: EvalTestCase) -> str:
+        """Generate evaluation prompt with CoT steps"""
+        steps_text = "\n".join(
+            [f"{i+1}. {step}" for i, step in enumerate(evaluation_steps)])
+
+        parts = [
+            f"User Input:\n{test_case.input}",
+            f"Model Output:\n{test_case.actual_output}"
+        ]
+
+        if test_case.expected_output:
+            parts.append(f"Expected Output:\n{test_case.expected_output}")
+
+        if test_case.retrieval_context:
+            parts.append(f"Context:\n" +
+                         "\n".join(test_case.retrieval_context))
+
+        input_block = "\n\n".join(parts)
+
+        return f"""You are a strict evaluator. Use the criteria and evaluation steps below.
+
+Evaluation Criteria:
+{criteria}
+
+Evaluation Steps:
+{steps_text}
+
+{input_block}
+
+Based on the evaluation steps, assign a score from 0.0 to 1.0 (where 0.0 is worst and 1.0 is best).
+
+**
+Return ONLY JSON:
+{{
+  "score": <float between 0.0 and 1.0>
+}}
+**
+
+JSON:"""
+
+    @staticmethod
+    def _prompt_reason(
+        criteria: str,
+        evaluation_steps: List[str],
+        test_case: EvalTestCase,
+        score: float
+    ) -> str:
+        """Generate explanation for the score"""
+        steps_text = "\n".join(
+            [f"{i+1}. {step}" for i, step in enumerate(evaluation_steps)])
+
+        parts = [
+            f"User Input:\n{test_case.input}",
+            f"Model Output:\n{test_case.actual_output}"
+        ]
+
+        if test_case.expected_output:
+            parts.append(f"Expected Output:\n{test_case.expected_output}")
+
+        if test_case.retrieval_context:
+            parts.append(f"Context:\n" +
+                         "\n".join(test_case.retrieval_context))
+
+        input_block = "\n\n".join(parts)
+
+        return f"""You assigned a score of {score:.2f} (out of 1.0) for this evaluation.
+
+Evaluation Criteria:
+{criteria}
+
+Evaluation Steps:
+{steps_text}
+
+{input_block}
+
+Final Score: {score:.2f}/1.0
+
+Explain why this score was assigned, referencing specific aspects from the evaluation steps.
+
+**
+Return ONLY JSON:
+{{
+  "reason": "Your explanation..."
+}}
+**
+
+JSON:"""
+
+    # ==================== HELPER METHODS ====================
+
+    def _extract_score_from_response(self, text: str) -> float:
+        """Extract float score from LLM response (0.0-1.0 range)"""
+        text = text.strip()
+
+        # Try JSON parsing first
+        try:
+            data = json.loads(extract_json_block(text))
+            if "score" in data:
+                score = float(data["score"])
+                if 0.0 <= score <= 1.0:
+                    return score
+        except:
+            pass
+
+        # Try regex patterns
+        patterns = [
+            r'"score"\s*:\s*(\d+\.?\d*)',
+            r'score[:\s]+(\d+\.?\d*)',
+            r'^\s*(\d+\.?\d*)\s*$',
+        ]
+
+        for pattern in patterns:
+            match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE)
+            if match:
+                score = float(match.group(1))
+                if 0.0 <= score <= 1.0:
+                    return score
+
+        raise RuntimeError(f"Failed to extract score from response: {text}")
+
+    # ==================== CORE ALGORITHM ====================
+
+    async def _probability_weighted_scoring(
+        self,
+        prompt: str,
+        n_samples: int = 20,
+        temperature: float = 2.0
+    ) -> Tuple[float, List[float], float]:
+        """
+        Probability-weighted scoring: score = Σ p(si) × si
+        Samples multiple times to estimate probability distribution
+
+        Returns:
+            (final_score, sampled_scores, total_cost)
+        """
+        total_cost = 0.0
+        scores = []
+
+        # Sample n times with high temperature
+        for _ in range(n_samples):
+            text, cost = await chat_complete(
+                self.model,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=temperature
+            )
+            total_cost += cost or 0.0
+
+            try:
+                score = self._extract_score_from_response(text)
+                scores.append(score)
+            except:
+                continue
+
+        if not scores:
+            raise RuntimeError(
+                f"Failed to extract any valid scores from {n_samples} samples")
+
+        # Calculate probability-weighted score as mean
+        weighted_score = sum(scores) / len(scores)
+
+        return weighted_score, scores, total_cost
+
+    async def evaluate(self, test_case: EvalTestCase) -> Dict[str, Any]:
+        """
+        Evaluate using Chain-of-Thought and Probability-Weighted Scoring.
+
+        Algorithm:
+        1. Auto-generate evaluation steps from criteria (CoT)
+        2. Apply probability-weighted scoring (20 samples, temp=2.0)
+        3. Generate detailed explanation
+        4. Build comprehensive evaluation_log
+        """
+        total_cost = 0.0
+
+        # Step 1: Auto-generate evaluation steps (Chain-of-Thought from G-Eval)
+        steps_prompt = self._prompt_generate_steps(self.criteria)
+        steps_text, step_cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": steps_prompt}],
+            temperature=0.0
+        )
+        total_cost += step_cost or 0.0
+
+        try:
+            parsed_steps = json.loads(extract_json_block(steps_text))
+            evaluation_steps = parsed_steps["steps"]
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to parse evaluation steps: {e}\n{steps_text}")
+
+        # Step 2: Generate evaluation prompt with CoT
+        eval_prompt = self._prompt_evaluate(
+            self.criteria, evaluation_steps, test_case)
+
+        # Step 3: Probability-weighted scoring (20 samples from G-Eval)
+        final_score, sampled_scores, scoring_cost = await self._probability_weighted_scoring(
+            eval_prompt,
+            n_samples=20,
+            temperature=2.0
+        )
+        total_cost += scoring_cost
+
+        # Step 4: Generate explanation
+        reason_prompt = self._prompt_reason(
+            self.criteria, evaluation_steps, test_case, final_score)
+        reason_text, reason_cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": reason_prompt}],
+            temperature=0.0
+        )
+        total_cost += reason_cost or 0.0
+
+        # Parse reason
+        try:
+            reason_data = json.loads(extract_json_block(reason_text))
+            reason = reason_data.get("reason", reason_text)
+        except:
+            reason = reason_text.strip()
+
+        success = final_score >= self.threshold
+
+        # Step 5: Build comprehensive evaluation_log
+        evaluation_log = {
+            "input_question": test_case.input,
+            "actual_output": test_case.actual_output,
+            "expected_output": test_case.expected_output,
+            "retrieval_context": test_case.retrieval_context,
+            "criteria": self.criteria,
+            "comment_criteria": "Custom evaluation criteria provided by user.",
+            "evaluation_steps": evaluation_steps,
+            "comment_evaluation_steps": "Auto-generated evaluation steps using Chain-of-Thought (CoT) technique from G-Eval.",
+            "sampled_scores": sampled_scores,
+            "comment_sampled_scores": f"Individual scores from {len(sampled_scores)} samples with temperature=2.0.",
+            "score_distribution": {f"{s:.2f}": sampled_scores.count(s) for s in set(sampled_scores)},
+            "comment_score_distribution": "Frequency distribution of sampled scores for probability-weighted calculation.",
+            "final_score": round(final_score, 4),
+            "comment_final_score": "Probability-weighted score calculated as mean of sampled scores (G-Eval technique).",
+            "threshold": self.threshold,
+            "success": success,
+            "comment_success": "Whether the final score passes the custom threshold.",
+            "final_reason": reason,
+            "comment_reasoning": "LLM-generated explanation based on evaluation steps and criteria."
+        }
+
+        return {
+            "score": round(final_score, 4),
+            "success": success,
+            "reason": reason,
+            "evaluation_cost": round(total_cost, 6),
+            "evaluation_log": evaluation_log,
+        }
+
+    @property
+    def name(self):
+        return f"Custom: {self.custom_name}"

eval_lib/metrics/faithfulness_metric/faithfulness.py

@@ -0,0 +1,140 @@
+# faithfulness_metric.py
+'''
+Faithfulness Metric: Evaluates the factual consistency of a chatbot's answer
+with respect to the retrieved context.
+Score calculation: Softmax aggregation of verdicts on factual statements
+'''
+from typing import List, Dict, Tuple, Any
+import json
+import re
+import numpy as np
+from math import exp
+from eval_lib.testcases_schema import EvalTestCase
+from eval_lib.metric_pattern import MetricPattern
+from eval_lib.llm_client import chat_complete
+from eval_lib.utils import score_agg, extract_json_block
+
+VERDICT_WEIGHTS = {
+    "fully": 1.0,
+    "mostly": 0.9,
+    "partial": 0.7,
+    "minor": 0.3,
+    "none": 0.0,
+}
+
+
+class FaithfulnessMetric(MetricPattern):
+    name = "faithfulnessMetric"
+
+    def __init__(
+            self,
+            model: str,
+            threshold: float = 0.7,
+            temperature: float = 0.5,
+    ):
+        super().__init__(model=model, threshold=threshold)
+        self.temperature = temperature
+
+    async def _generate_statements(self, answer: str) -> Tuple[List[str], float]:
+        prompt = (
+            "Extract standalone factual claims from the following answer.\n"
+            "Each statement must be a distinct, verifiable fact.\n\n"
+            f"Answer:\n{answer}\n\n"
+            "Return a JSON array of strings."
+        )
+        text, cost = await chat_complete(self.model, [{"role": "user", "content": prompt}], temperature=0.0)
+        raw_json = extract_json_block(text)
+        statements = json.loads(raw_json)
+        assert isinstance(statements, list)
+        return statements, cost or 0.0
+
+    async def _generate_verdicts(self, context: str, statements: List[str]) -> Tuple[List[Dict[str, str]], float, float]:
+        prompt = (
+            "Evaluate how well each statement is supported by the context.\n\n"
+            "Levels:\n"
+            "- fully: directly supported word-for-word\n"
+            "- mostly: strongly supported but wording differs slightly\n"
+            "- partial: partially supported but with some gaps\n"
+            "- minor: tangentially related or ambiguous\n"
+            "- none: clearly unsupported or contradicted\n\n"
+            f"CONTEXT:\n{context}\n\n"
+            f"STATEMENTS (JSON array):\n{json.dumps(statements, ensure_ascii=False)}\n\n"
+            "Return only a JSON array of objects like:\n"
+            '[{"verdict": "fully|mostly|partial|minor|none", '
+            '"reason": "<brief>", '
+            '"support": "<exact context sentence(s)> or \'none\'"}]'
+        )
+        text, cost = await chat_complete(self.model, [{"role": "user", "content": prompt}], temperature=0.0)
+        raw_json = extract_json_block(text)
+        verdicts: List[Dict[str, Any]] = json.loads(raw_json)
+
+        for v in verdicts:
+            supp = v.get("support", "").strip().lower()
+            if supp == "none" and v["verdict"] in ("fully", "mostly"):
+                v["verdict"] = "partial"
+
+        scores = [VERDICT_WEIGHTS.get(v["verdict"], 0.0) for v in verdicts]
+        score = round(score_agg(scores, temperature=self.temperature), 4)
+        return verdicts, score, cost or 0.0
+
+    async def _summarize_reasons_via_llm(self, verdicts: List[Dict[str, str]]) -> Tuple[str, float]:
+        grouped: Dict[str, List[str]] = {}
+        for v in verdicts:
+            grouped.setdefault(v["verdict"], []).append(v["reason"])
+        bullets = []
+        for tag in ("fully", "mostly", "partial", "none"):
+            bullets.extend(f"- {r}" for r in grouped.get(tag, [])[:2])
+        prompt = (
+            "Summarize the following points from a factual consistency evaluation.\n"
+            "Give one short paragraph (1-2 sentences) that explains whether the answer "
+            "was well supported by the context, mentioning both strong and weak parts.\n\n"
+            f"{chr(10).join(bullets)}\n\n"
+            "Summary:"
+        )
+        text, cost = await chat_complete(self.model, [{"role": "user", "content": prompt}], temperature=0.0)
+        return text.strip(), cost or 0.0
+
+    async def evaluate(self, test_case: EvalTestCase) -> Dict[str, any]:
+        llm_cost = 0.0
+        answer = test_case.actual_output
+        context = "\n".join(test_case.retrieval_context or [])
+        question = test_case.input
+
+        # 1. Statements from answer
+        statements, cost = await self._generate_statements(answer)
+        llm_cost += cost
+
+        # 2. Verdicts against context
+        verdicts, verdict_score, cost = await self._generate_verdicts(context, statements)
+        llm_cost += cost
+
+        # 3. Reason summary
+        summary_reason, cost = await self._summarize_reasons_via_llm(verdicts)
+        llm_cost += cost
+
+        success = verdict_score >= self.threshold
+
+        evaluation_log = {
+            "input_question": question,
+            "retrieval_context": test_case.retrieval_context,
+            "answer": answer,
+            "statements": statements,
+            "comment_statements": "Factual assertions extracted from the answer.",
+            "verdicts": verdicts,
+            "comment_verdicts": "Each verdict shows how well a statement is supported by the context.",
+            "final_score": verdict_score,
+            "comment_final_score": "Final score based on faithfulness of statements.",
+            "threshold": self.threshold,
+            "success": success,
+            "comment_success": "Whether the score meets the required threshold.",
+            "final_reason": summary_reason,
+            "comment_reasoning": "Summary explanation based on all verdicts."
+        }
+
+        return {
+            "score": verdict_score,
+            "success": success,
+            "reason": summary_reason,
+            "evaluation_cost": round(llm_cost, 6),
+            "evaluation_log": evaluation_log
+        }