eval-ai-library 0.1.0__py3-none-any.whl

eval_ai_library-0.1.0.dist-info/RECORD

@@ -0,0 +1,34 @@
+eval_ai_library-0.1.0.dist-info/licenses/LICENSE,sha256=rK9uLDgWNrCHNdp-Zma_XghDE7Fs0u0kDi3WMcmYx6w,1074
+eval_lib/__init__.py,sha256=vwyPEMhyQqJR8_usT-LzI1xjvFTBa8S16s53frz4JJM,2546
+eval_lib/evaluate.py,sha256=XwtNkEaVzbrB2m6rgRaeFcb9kgHeHjAMM-WxMGCNs8E,12516
+eval_lib/evaluation_schema.py,sha256=7IDd_uozqewhh7k0p1hKut_20udvRxxkV6thclxKUg0,1904
+eval_lib/llm_client.py,sha256=y-6blKiMtZcPwRGeQ-_XHMIcAtgI-JxH3sArGxYErbI,7620
+eval_lib/metric_pattern.py,sha256=7EL17zMqScjJ_NbgwmdwX_VBjrUq28SJnnBL8bfwO9Y,8151
+eval_lib/price.py,sha256=jbmkkUTxPuXrkSHuaJYPl7jSzfDIzQ9p_swWWs26UJ0,1986
+eval_lib/py.typed,sha256=8PjyZ1aVoQpRVvt71muvuq5qE-jTFZkK-GLHkhdebmc,26
+eval_lib/testcases_schema.py,sha256=qI4o6kX0jH1DR3sHGXUnu3Cyt2oq7rGlsMlOaXSt6F4,696
+eval_lib/utils.py,sha256=KNqNtZ3YO09Pf7alN7sT2P81xEGqii0AiNnJyN5jTlA,2744
+eval_lib/agent_metrics/__init__.py,sha256=20Y4BsicD2s7OkOBQPBvB2JKStBDtplv52_q6q35Vgo,525
+eval_lib/agent_metrics/knowledge_retention_metric/knowledge_retention.py,sha256=ensJZwq7N5DOm0VM4xmEjEsCkIxFytcf1GnoDJt8nBQ,8085
+eval_lib/agent_metrics/role_adherence_metric/role_adherence.py,sha256=llH28rR2fasWqL-KoovsnPdqcnUHRy5tINm7Q_6dY5E,8922
+eval_lib/agent_metrics/task_success_metric/task_success_rate.py,sha256=g2I0yQ5oB1TJ33LH-WpY4xa2Igz8dN36AghU47pjOFY,12379
+eval_lib/agent_metrics/tools_correctness_metric/tool_correctness.py,sha256=bXFDPAe3MBDxKGliSeiyVCZ6v2sYksGFUUmO_dpEI60,4081
+eval_lib/datagenerator/datagenerator.py,sha256=AClljFk-ZLKLkeaDrfRVE1-7pixiC60fwsoAXXVsXEY,8200
+eval_lib/datagenerator/document_loader.py,sha256=bx3yvpH8pLlgbwPZRSLQt0Kl-xT6eI2AqRApIPyMYbI,17554
+eval_lib/datagenerator/prompts.py,sha256=iQAYitAbLud3vWJnXGh_OCF4syWYS_S71zZEPI6qYAU,7213
+eval_lib/metrics/__init__.py,sha256=3qClCCjPXt5i0c38g5krfuQnqlAXEl-jhAHy1C_ICMY,1213
+eval_lib/metrics/answer_precision_metric/answer_precision.py,sha256=9EKgp_4Kw01nSRL5lyHxbHSTI1mQUnFpEtOWUf51aA0,14909
+eval_lib/metrics/answer_relevancy_metric/answer_relevancy.py,sha256=fRSeJhOQAV7FJHaDbQQjTXABIbKRO47-WMKMgPqzIZ0,8340
+eval_lib/metrics/bias_metric/bias.py,sha256=HwabG0yyV4U_LXIsdviawKoqbM4QZxA3fcSnJNMpVeM,3788
+eval_lib/metrics/contextual_precision_metric/contextual_precision.py,sha256=YZSfMoMwoVgu_5ZSp_yxpazbXM_9d-bWi74obPIhQ68,3916
+eval_lib/metrics/contextual_recall_metric/contextual_recall.py,sha256=bZQE6AZpG_dnjrNjThhf9k9X0GtaVg3zPTvqQdXRjgQ,3783
+eval_lib/metrics/contextual_relevancy_metric/contextual_relevancy.py,sha256=aYle5W5NtcHuvumBE4k1O7eyREU-dILtoTl_fHAWdNs,6359
+eval_lib/metrics/custom_metric/custom_eval.py,sha256=8YzbS45LQ0XtwIwyxnzFP5ujYeKPG_NWdWD0TnTwm2E,9737
+eval_lib/metrics/faithfulness_metric/faithfulness.py,sha256=jHZMYflkXMEDsh8czENPisitTmaplI7Klr2PoPxE1Rw,5789
+eval_lib/metrics/geval/geval.py,sha256=MhWd4KpSIsTEvanscuLsIXOtjOGEtw_2-8ctrqabzTo,10490
+eval_lib/metrics/restricted_refusal_metric/restricted_refusal.py,sha256=TQpoiN1hG2KKw1ILpJsmoE2T8uMGhxHz3wswDaqcBPA,4156
+eval_lib/metrics/toxicity_metric/toxicity.py,sha256=Rh5UPSeAtkGopmCU4JGTAVLHbgFKrhVO83pl6vdNdTw,3882
+eval_ai_library-0.1.0.dist-info/METADATA,sha256=dJ2lV7VKlvVfmcIimgIAfgi900g21DBayknvs5Ng8yg,23834
+eval_ai_library-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+eval_ai_library-0.1.0.dist-info/top_level.txt,sha256=uQHpEd2XI0oZgq1eCww9zMvVgDJgwXMWkCD45fYUzEg,9
+eval_ai_library-0.1.0.dist-info/RECORD,,

eval_ai_library-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

eval_ai_library-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Aleksandr Meshkov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

eval_ai_library-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+eval_lib

eval_lib/__init__.py

@@ -0,0 +1,122 @@
+"""
+Eval AI Library - Comprehensive AI Model Evaluation Framework
+
+A powerful library for evaluating AI models with support for multiple LLM providers
+and a wide range of evaluation metrics for RAG systems and AI agents.
+"""
+
+__version__ = "0.1.0"
+__author__ = "Aleksandr Meskov"
+
+# Core evaluation functions
+from eval_lib.evaluate import evaluate, evaluate_conversations
+from eval_lib.utils import score_agg, softmax_agg
+
+# Test case schemas
+from eval_lib.testcases_schema import (
+    EvalTestCase,
+    ConversationalEvalTestCase,
+    ToolCall
+)
+
+# Evaluation schemas
+from eval_lib.evaluation_schema import (
+    MetricResult,
+    TestCaseResult,
+    ConversationalTestCaseResult
+)
+
+# Base patterns
+from eval_lib.metric_pattern import (
+    MetricPattern,
+    ConversationalMetricPattern
+)
+
+# LLM client
+from eval_lib.llm_client import (
+    chat_complete,
+    get_embeddings,
+    LLMDescriptor,
+    Provider
+)
+
+# RAG Metrics
+from eval_lib.metrics import (
+    AnswerRelevancyMetric,
+    AnswerPrecisionMetric,
+    FaithfulnessMetric,
+    ContextualRelevancyMetric,
+    ContextualPrecisionMetric,
+    ContextualRecallMetric,
+    BiasMetric,
+    ToxicityMetric,
+    RestrictedRefusalMetric,
+    GEval,
+    CustomEvalMetric
+)
+
+# Agent Metrics
+from eval_lib.agent_metrics import (
+    ToolCorrectnessMetric,
+    TaskSuccessRateMetric,
+    RoleAdherenceMetric,
+    KnowledgeRetentionMetric
+)
+
+# Data generator
+from eval_lib.datagenerator.datagenerator import DataGenerator
+from eval_lib.datagenerator.document_loader import DocumentLoader
+
+__all__ = [
+    # Version
+    "__version__",
+
+    # Core functions
+    "evaluate",
+    "evaluate_conversations",
+
+    # Schemas
+    "EvalTestCase",
+    "ConversationalEvalTestCase",
+    "ToolCall",
+    "MetricResult",
+    "TestCaseResult",
+    "ConversationalTestCaseResult",
+
+    # Patterns
+    "MetricPattern",
+    "ConversationalMetricPattern",
+
+    # LLM
+    "chat_complete",
+    "get_embeddings",
+    "LLMDescriptor",
+    "Provider",
+
+    # RAG Metrics
+    "AnswerRelevancyMetric",
+    "AnswerPrecisionMetric",
+    "FaithfulnessMetric",
+    "ContextualRelevancyMetric",
+    "ContextualPrecisionMetric",
+    "ContextualRecallMetric",
+    "BiasMetric",
+    "ToxicityMetric",
+    "RestrictedRefusalMetric",
+    "GEval",
+    "CustomEvalMetric",
+
+    # Agent Metrics
+    "ToolCorrectnessMetric",
+    "TaskSuccessRateMetric",
+    "RoleAdherenceMetric",
+    "KnowledgeRetentionMetric",
+
+    # Data Generation
+    "DataGenerator",
+    "DocumentLoader",
+
+    # Utils
+    "score_agg",
+    "extract_json_block",
+]

eval_lib/agent_metrics/__init__.py

@@ -0,0 +1,12 @@
+from eval_lib.agent_metrics.tools_correctness_metric.tool_correctness import ToolCorrectnessMetric
+from eval_lib.agent_metrics.task_success_metric.task_success_rate import TaskSuccessRateMetric
+from eval_lib.agent_metrics.role_adherence_metric.role_adherence import RoleAdherenceMetric
+from eval_lib.agent_metrics.knowledge_retention_metric.knowledge_retention import KnowledgeRetentionMetric
+
+
+__all__ = [
+    "ToolCorrectnessMetric",
+    "TaskSuccessRateMetric",
+    "RoleAdherenceMetric",
+    "KnowledgeRetentionMetric"
+]

eval_lib/agent_metrics/knowledge_retention_metric/knowledge_retention.py

@@ -0,0 +1,231 @@
+# knowledge_retention.py
+"""
+Knowledge Retention Metric: Evaluates how well the assistant remembers and maintains
+context across a multi-turn conversation.
+
+Score calculation: Softmax aggregation of retention verdicts
+"""
+import json
+from typing import List, Dict, Any, Tuple
+from eval_lib.testcases_schema import ConversationalEvalTestCase
+from eval_lib.metric_pattern import ConversationalMetricPattern
+from eval_lib.llm_client import chat_complete
+from eval_lib.utils import score_agg, extract_json_block
+
+
+# Verdict weights for knowledge retention levels
+VERDICT_WEIGHTS = {
+    "fully": 1.0,      # Remembers every relevant fact
+    "mostly": 0.9,     # Slight omissions, no contradiction
+    "partial": 0.7,    # Several lapses but overall context kept
+    "minor": 0.3,      # One small lapse or omission
+    "none": 0.0        # Assistant contradicts or forgets previous facts
+}
+
+
+class KnowledgeRetentionMetric(ConversationalMetricPattern):
+    """
+    Evaluates how well an AI assistant retains and recalls information
+    shared across multiple conversation turns.
+    """
+
+    name = "knowledgeRetentionMetric"
+
+    def __init__(
+        self,
+        model: str,
+        threshold: float = 0.7,
+        temperature: float = 0.5,
+    ):
+        """
+        Initialize Knowledge Retention metric.
+
+        Args:
+            model: LLM model name
+            threshold: Success threshold (0.0-1.0)
+            temperature: Score aggregation temperature for softmax
+        """
+        super().__init__(model=model, threshold=threshold)
+        self.temperature = temperature
+
+    # ==================== HELPER METHODS ====================
+
+    @staticmethod
+    def _render_dialogue(turns) -> str:
+        """Convert conversation turns into readable format"""
+        return "\n".join(
+            f"{i+1}. User: {t.input}\n   Assistant: {t.actual_output}"
+            for i, t in enumerate(turns)
+        )
+
+    @staticmethod
+    def _prompt_label_help() -> str:
+        """Explanation of retention verdict levels"""
+        return """Rate knowledge retention (worst → best):
+
+none   – assistant contradicts or forgets previous facts  
+minor  – one small lapse or omission  
+partial– several lapses but overall context kept  
+mostly – slight omissions, no contradiction  
+fully  – remembers every relevant fact"""
+
+    @staticmethod
+    def _prompt_few_shot() -> str:
+        """Few-shot examples for verdict generation"""
+        return """Example GOOD:
+Conversation:
+1. User: What year was Python created?
+   Assistant: Python was first released in 1991.
+2. User: Remind me, who created it?
+   Assistant: It was created by Guido van Rossum in 1991.
+Verdicts:
+[{"verdict":"fully","reason":"Assistant repeated year and author correctly"}]
+
+Example BAD:
+Conversation:
+1. User: I live in Spain.
+   Assistant: Great! How's the weather?
+2. User: Remind me later that I live in Spain.
+   Assistant: Sure, I'll remind you that you live in Italy.
+Verdicts:
+[{"verdict":"none","reason":"Assistant contradicted the country"}]"""
+
+    # ==================== CORE EVALUATION STEPS ====================
+
+    async def _generate_verdicts(
+        self,
+        dialogue: str
+    ) -> Tuple[List[Dict[str, str]], float, float]:
+        """
+        Generate retention verdicts for the conversation.
+
+        Args:
+            dialogue: Formatted conversation text
+
+        Returns:
+            Tuple of (verdicts_list, aggregated_score, llm_cost)
+        """
+        prompt = (
+            f"{self._prompt_label_help()}\n\n"
+            f"{self._prompt_few_shot()}\n\n"
+            "Now analyse the next conversation.\n\n"
+            f"Conversation:\n{dialogue}\n\n"
+            "Return ONE JSON array with 1 object in the form:\n"
+            "[{\"verdict\":\"fully|mostly|partial|minor|none\",\"reason\":\"…\"}]"
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        try:
+            raw_json = extract_json_block(text)
+            verdicts = json.loads(raw_json)
+
+            if not isinstance(verdicts, list):
+                raise ValueError("Expected JSON array of verdicts")
+
+            # Calculate aggregated score from verdicts
+            weights = [VERDICT_WEIGHTS.get(
+                v.get("verdict", "none"), 0.0) for v in verdicts]
+            score = round(score_agg(weights, temperature=self.temperature), 4)
+
+            return verdicts, score, cost or 0.0
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to parse verdicts: {e}\n{text}")
+
+    async def _summarize_verdicts(
+        self,
+        verdicts: List[Dict[str, str]]
+    ) -> Tuple[str, float]:
+        """
+        Generate concise summary of retention assessment.
+
+        Args:
+            verdicts: List of verdict objects with reasons
+
+        Returns:
+            Tuple of (summary_text, llm_cost)
+        """
+        bullets = "\n".join(f"- {v['reason']}" for v in verdicts)
+
+        prompt = (
+            "Write a concise (max 2 sentences) summary of the assistant's knowledge retention, "
+            "based on these points:\n\n"
+            f"{bullets}\n\n"
+            "Summary:"
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        return text.strip(), cost or 0.0
+
+    # ==================== MAIN EVALUATION ====================
+
+    async def evaluate(self, test_case: ConversationalEvalTestCase) -> Dict[str, Any]:
+        """
+        Evaluate knowledge retention across conversation turns.
+
+        Steps:
+        1. Format dialogue into readable text
+        2. Generate retention verdicts (fully/mostly/partial/minor/none)
+        3. Aggregate verdicts into final score using softmax
+        4. Generate summary explanation
+        5. Build comprehensive evaluation log
+
+        Args:
+            test_case: Conversational test case with multiple turns
+
+        Returns:
+            Evaluation results with score, success, reason, cost, and detailed log
+        """
+        total_cost = 0.0
+
+        # Step 1: Format dialogue
+        dialogue_text = self._render_dialogue(test_case.turns)
+
+        # Step 2: Generate retention verdicts
+        verdicts, verdict_score, cost = await self._generate_verdicts(dialogue_text)
+        total_cost += cost
+
+        # Step 3: Generate summary explanation
+        summary, cost = await self._summarize_verdicts(verdicts)
+        total_cost += cost
+
+        # Step 4: Determine success
+        final_score = verdict_score
+        success = final_score >= self.threshold
+
+        # Step 5: Build evaluation log
+        evaluation_log = {
+            "dialogue": dialogue_text,
+            "comment_dialogue": "Full conversation text used for retention evaluation.",
+            "number_of_turns": len(test_case.turns),
+            "comment_number_of_turns": "Total conversation turns analyzed.",
+            "verdicts": verdicts,
+            "comment_verdicts": "LLM-generated verdicts assessing retention level (fully/mostly/partial/minor/none).",
+            "verdict_weights": {v["verdict"]: VERDICT_WEIGHTS.get(v["verdict"], 0.0) for v in verdicts},
+            "comment_verdict_weights": "Numeric weights assigned to each verdict for score calculation.",
+            "final_score": final_score,
+            "comment_final_score": f"Softmax aggregation of verdict weights (temperature={self.temperature}).",
+            "threshold": self.threshold,
+            "success": success,
+            "comment_success": "Whether the retention score meets the required threshold.",
+            "final_reason": summary,
+            "comment_reasoning": "Concise explanation of the assistant's knowledge retention performance."
+        }
+
+        return {
+            "score": final_score,
+            "success": success,
+            "reason": summary,
+            "evaluation_cost": round(total_cost, 6),
+            "evaluation_log": evaluation_log
+        }

eval_lib/agent_metrics/role_adherence_metric/role_adherence.py

@@ -0,0 +1,251 @@
+# role_adherence.py
+"""
+Role Adherence Metric: Evaluates how well the AI assistant maintains its assigned
+role and character throughout a multi-turn conversation.
+
+Score calculation: Softmax aggregation of adherence verdicts
+"""
+import json
+from typing import List, Dict, Any, Tuple
+from eval_lib.testcases_schema import ConversationalEvalTestCase
+from eval_lib.metric_pattern import ConversationalMetricPattern
+from eval_lib.llm_client import chat_complete
+from eval_lib.utils import score_agg, extract_json_block
+
+
+# Verdict weights for role adherence levels
+VERDICT_WEIGHTS = {
+    "fully": 1.0,      # Perfectly maintains role throughout
+    "mostly": 0.9,     # Minor deviations but stays in character
+    "partial": 0.7,    # Some role breaks but generally consistent
+    "minor": 0.3,      # Frequently breaks character
+    "none": 0.0        # Completely ignores assigned role
+}
+
+
+class RoleAdherenceMetric(ConversationalMetricPattern):
+    """
+    Evaluates how consistently an AI assistant adheres to its assigned role
+    across multiple conversation turns.
+    """
+
+    name = "roleAdherenceMetric"
+
+    def __init__(
+        self,
+        model: str,
+        threshold: float = 0.7,
+        temperature: float = 0.5,
+    ):
+        """
+        Initialize Role Adherence metric.
+
+        Args:
+            model: LLM model name
+            threshold: Success threshold (0.0-1.0)
+            temperature: Score aggregation temperature for softmax
+        """
+        super().__init__(model=model, threshold=threshold)
+        self.temperature = temperature
+
+    # ==================== HELPER METHODS ====================
+
+    @staticmethod
+    def _render_dialogue(turns) -> str:
+        """Convert conversation turns into readable format"""
+        return "\n".join(
+            f"{i+1}. User: {t.input}\n   Assistant: {t.actual_output}"
+            for i, t in enumerate(turns)
+        )
+
+    @staticmethod
+    def _prompt_label_help() -> str:
+        """Explanation of role adherence verdict levels"""
+        return """Rate role adherence (worst → best):
+
+none    – completely ignores assigned role
+minor   – frequently breaks character
+partial – some role breaks but generally consistent
+mostly  – minor deviations but stays in character
+fully   – perfectly maintains role throughout"""
+
+    @staticmethod
+    def _prompt_few_shot() -> str:
+        """Few-shot examples for verdict generation"""
+        return """Example GOOD:
+Role: You are a professional medical advisor. Be formal and evidence-based.
+Conversation:
+1. User: What causes headaches?
+   Assistant: Headaches can result from various factors including dehydration, stress, or underlying medical conditions. I recommend consulting a healthcare provider for persistent symptoms.
+Verdicts:
+[{"verdict":"fully","reason":"Maintained formal medical tone and evidence-based approach"}]
+
+Example BAD:
+Role: You are a professional financial advisor. Use formal language.
+Conversation:
+1. User: Should I invest in stocks?
+   Assistant: Yo dude! Stocks are totally rad! Just YOLO into them lol 🚀
+Verdicts:
+[{"verdict":"none","reason":"Completely abandoned professional tone and formal language requirement"}]"""
+
+    # ==================== CORE EVALUATION STEPS ====================
+
+    async def _generate_verdicts(
+        self,
+        role_description: str,
+        dialogue_text: str
+    ) -> Tuple[List[Dict[str, str]], float, float]:
+        """
+        Generate role adherence verdicts for each conversation turn.
+
+        Args:
+            role_description: The assigned role/character description
+            dialogue_text: Formatted conversation text
+
+        Returns:
+            Tuple of (verdicts_list, aggregated_score, llm_cost)
+        """
+        prompt = f"""{self._prompt_label_help()}
+
+{self._prompt_few_shot()}
+
+Now evaluate the following conversation.
+
+ASSIGNED ROLE:
+{role_description}
+
+DIALOGUE:
+{dialogue_text}
+
+Task: Judge how well the assistant stays in character throughout the conversation.
+
+For each assistant reply, assign a verdict: "fully", "mostly", "partial", "minor", or "none".
+
+Return JSON array:
+[{{"verdict": "fully|mostly|partial|minor|none", "reason": "<explanation>"}}, ...]"""
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        try:
+            raw_json = extract_json_block(text)
+            verdicts = json.loads(raw_json)
+
+            if not isinstance(verdicts, list):
+                raise ValueError("Expected JSON array of verdicts")
+
+            # Calculate aggregated score from verdicts
+            weights = [VERDICT_WEIGHTS.get(
+                v.get("verdict", "none"), 0.0) for v in verdicts]
+            score = round(score_agg(weights, temperature=self.temperature), 4)
+
+            return verdicts, score, cost or 0.0
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to parse verdicts: {e}\n{text}")
+
+    async def _summarize_verdicts(
+        self,
+        verdicts: List[Dict[str, str]]
+    ) -> Tuple[str, float]:
+        """
+        Generate concise summary of role adherence assessment.
+
+        Args:
+            verdicts: List of verdict objects with reasons
+
+        Returns:
+            Tuple of (summary_text, llm_cost)
+        """
+        # Take up to 6 most relevant verdicts for summary
+        bullets = "\n".join(f"- {v['reason']}" for v in verdicts[:6])
+
+        prompt = (
+            "Write a concise (max 2 sentences) summary of how well the chatbot stayed in character, "
+            "based on these observations:\n\n"
+            f"{bullets}\n\n"
+            "Summary:"
+        )
+
+        text, cost = await chat_complete(
+            self.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0
+        )
+
+        return text.strip(), cost or 0.0
+
+    # ==================== MAIN EVALUATION ====================
+
+    async def evaluate(self, test_case: ConversationalEvalTestCase) -> Dict[str, Any]:
+        """
+        Evaluate role adherence across conversation turns.
+
+        Steps:
+        1. Extract chatbot role from test case
+        2. Format dialogue into readable text
+        3. Generate adherence verdicts (fully/mostly/partial/minor/none)
+        4. Aggregate verdicts into final score using softmax
+        5. Generate summary explanation
+        6. Build comprehensive evaluation log
+
+        Args:
+            test_case: Conversational test case with multiple turns and chatbot role
+
+        Returns:
+            Evaluation results with score, success, reason, cost, and detailed log
+        """
+        total_cost = 0.0
+
+        # Step 1: Extract role
+        role_description = test_case.chatbot_role or "No role specified"
+
+        # Step 2: Format dialogue
+        dialogue_text = self._render_dialogue(test_case.turns)
+
+        # Step 3: Generate role adherence verdicts
+        verdicts, verdict_score, cost = await self._generate_verdicts(
+            role_description,
+            dialogue_text
+        )
+        total_cost += cost
+
+        # Step 4: Generate summary explanation
+        summary, cost = await self._summarize_verdicts(verdicts)
+        total_cost += cost
+
+        # Step 5: Determine success
+        final_score = verdict_score
+        success = final_score >= self.threshold
+
+        # Step 6: Build evaluation log
+        evaluation_log = {
+            "chatbot_role": role_description,
+            "comment_chatbot_role": "The assigned role/character the assistant should maintain.",
+            "dialogue": dialogue_text,
+            "comment_dialogue": "Full conversation text used for role adherence evaluation.",
+            "number_of_turns": len(test_case.turns),
+            "comment_number_of_turns": "Total conversation turns analyzed.",
+            "verdicts": verdicts,
+            "comment_verdicts": "LLM-generated verdicts assessing adherence level per turn (fully/mostly/partial/minor/none).",
+            "verdict_weights": {v["verdict"]: VERDICT_WEIGHTS.get(v["verdict"], 0.0) for v in verdicts},
+            "comment_verdict_weights": "Numeric weights assigned to each verdict for score calculation.",
+            "final_score": final_score,
+            "comment_final_score": f"Softmax aggregation of verdict weights (temperature={self.temperature}).",
+            "threshold": self.threshold,
+            "success": success,
+            "comment_success": "Whether the role adherence score meets the required threshold.",
+            "final_reason": summary,
+            "comment_reasoning": "Concise explanation of how well the assistant maintained its assigned role."
+        }
+
+        return {
+            "score": final_score,
+            "success": success,
+            "reason": summary,
+            "evaluation_cost": round(total_cost, 6),
+            "evaluation_log": evaluation_log
+        }