eval-protocol 0.0.3__py3-none-any.whl

eval_protocol/rewards/multiple_choice_math_reward.py

@@ -0,0 +1,232 @@
+"""
+Multiple Choice Question (MCQ) reward function.
+
+This module provides a reward function specifically for evaluating
+answers to multiple-choice questions, where the answer is typically
+a single letter (e.g., A, B, C, D, E).
+"""
+
+import re
+from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union
+
+from ..models import EvaluateResult, Message, MetricResult
+from ..typed_interface import reward_function
+
+
+class MatchInfo(TypedDict):
+    text: str
+    letter: str
+    span: Tuple[int, int]
+    priority: int
+
+
+def extract_mcq_option(text: str) -> List[Tuple[str, str]]:
+    """
+    Extracts MCQ options (A-E) from text.
+    Prioritizes options in parentheses or brackets, or standalone letters.
+
+    Args:
+        text: The text to extract MCQ options from.
+
+    Returns:
+        A list of tuples, where each tuple contains the original matched
+        string and the uppercase letter of the MCQ option.
+        Returns an empty list if no MCQ option is confidently extracted.
+    """
+    mcq_answers: List[Tuple[str, str]] = []
+    found_mcq_letters = set()
+
+    patterns = [
+        r"(\(([A-E])\))",  # (A)
+        r"(\[([A-E])\])",  # [A]
+        r"(\{([A-E])\})",  # {A}
+        r"((?<![a-zA-Z0-9_])([A-E])\.(?!\w))",  # A.
+        r"((?<![a-zA-Z0-9_])([A-E])(?![a-zA-Z0-9_]))",  # Standalone A
+    ]
+
+    all_potential_matches: List[MatchInfo] = []
+    for p_idx, p_str in enumerate(patterns):
+        for match in re.finditer(p_str, text, re.IGNORECASE):
+            option_text = match.group(1)
+            letter = match.group(2)
+            span_tuple = match.span(1)
+            if not (
+                isinstance(span_tuple, tuple)
+                and len(span_tuple) == 2
+                and isinstance(span_tuple[0], int)
+                and isinstance(span_tuple[1], int)
+            ):
+                continue
+
+            match_data: MatchInfo = {
+                "text": option_text if option_text is not None else "",
+                "letter": letter.upper() if letter is not None else "",
+                "span": span_tuple,
+                "priority": p_idx,
+            }
+            all_potential_matches.append(match_data)
+
+    all_potential_matches.sort(key=lambda m: (m["span"][0], m["priority"], -(m["span"][1] - m["span"][0])))
+
+    last_covered_end = -1
+    for match_info in all_potential_matches:
+        start, end = match_info["span"]
+        if start >= last_covered_end:
+            letter_upper = match_info["letter"]
+            if letter_upper not in found_mcq_letters:
+                mcq_answers.append((match_info["text"], letter_upper))
+                found_mcq_letters.add(letter_upper)
+            last_covered_end = end
+    # Renamed final_mcq_answers to mcq_answers for clarity
+    return mcq_answers
+
+
+@reward_function  # type: ignore[arg-type]
+def multiple_choice_math_reward(
+    messages: List[Message],
+    ground_truth: List[Message],
+    **kwargs: Any,
+) -> EvaluateResult:
+    """
+    Evaluate multiple-choice answers in messages.
+
+    Extracts MCQ options (A-E) from the last assistant message in
+    the generated messages and from the ground truth assistant message, then compares them.
+
+    Args:
+        messages: Generated conversation messages, where the last message is the
+                  assistant's response.
+        ground_truth: A list containing the ground truth assistant message.
+        **kwargs: Additional keyword arguments.
+
+    Returns:
+        EvaluateResult with score and metrics.
+    """
+    metrics: Dict[str, MetricResult] = {}
+
+    if not messages:
+        return EvaluateResult(
+            score=0.0,
+            reason="Missing generated messages",
+            metrics={
+                "error": MetricResult(
+                    score=0.0,
+                    is_score_valid=False,
+                    reason="Missing generated messages",
+                )
+            },
+        )
+
+    if not ground_truth:
+        return EvaluateResult(
+            score=0.0,
+            reason="Missing ground truth message",
+            metrics={
+                "error": MetricResult(
+                    score=0.0,
+                    is_score_valid=False,
+                    reason="Missing ground truth message",
+                )
+            },
+        )
+
+    gen_content = ""
+    if messages and len(messages) > 0:
+        gen_response_message = messages[-1]
+        if gen_response_message.role == "assistant":
+            gen_content = gen_response_message.content or ""
+
+    if not gen_content:
+        metrics["error_generated_message"] = MetricResult(
+            score=0.0,
+            is_score_valid=False,
+            reason="Invalid generated message: Last message not from assistant or has no content.",
+        )
+        return EvaluateResult(
+            score=0.0,
+            reason="Last generated message not from assistant or has no content.",
+            metrics=metrics,
+        )
+
+    orig_content = ""
+    if ground_truth and len(ground_truth) > 0:
+        orig_response_message = ground_truth[0]
+        if orig_response_message.role == "assistant":
+            orig_content = orig_response_message.content or ""
+
+    if not orig_content:
+        metrics["error_original_message"] = MetricResult(
+            score=0.0,
+            is_score_valid=False,
+            reason="Invalid ground truth message: Not an assistant message or has no content.",
+        )
+        return EvaluateResult(
+            score=0.0,
+            reason="Invalid ground truth message: Not an assistant message or has no content.",
+            metrics=metrics,
+        )
+
+    gen_mcq_options = extract_mcq_option(gen_content)
+    orig_mcq_options = extract_mcq_option(orig_content)
+
+    def format_extracted_mcq(items: List[Tuple[str, str]]) -> str:
+        if not items:
+            return "None"
+        return ", ".join([f"'{i[0]}' ({i[1]})" for i in items])
+
+    metrics["extracted_original_mcq"] = MetricResult(
+        score=1.0 if orig_mcq_options else 0.0,
+        is_score_valid=bool(orig_mcq_options),
+        reason=f"Extracted from original: {format_extracted_mcq(orig_mcq_options)}",
+    )
+    metrics["extracted_generated_mcq"] = MetricResult(
+        score=1.0 if gen_mcq_options else 0.0,
+        is_score_valid=bool(gen_mcq_options),
+        reason=f"Extracted from generated: {format_extracted_mcq(gen_mcq_options)}",
+    )
+
+    if not orig_mcq_options:
+        return EvaluateResult(
+            score=0.0,
+            reason="Could not extract MCQ option from original message (ground truth). Assumed not an MCQ.",
+            metrics=metrics,
+        )
+
+    if not gen_mcq_options:
+        return EvaluateResult(
+            score=0.0,
+            reason="Could not extract MCQ option from generated message, but original message has an MCQ option.",
+            metrics=metrics,
+        )
+
+    # Ideally, MCQs should have one clear answer.
+    if len(orig_mcq_options) > 1:
+        metrics["ambiguous_original_mcq"] = MetricResult(
+            score=0.0,
+            is_score_valid=False,
+            reason=f"Original message has multiple MCQ options extracted: {format_extracted_mcq(orig_mcq_options)}",
+        )
+
+    if len(gen_mcq_options) > 1:
+        metrics["ambiguous_generated_mcq"] = MetricResult(
+            score=0.0,
+            is_score_valid=False,
+            reason=f"Generated message has multiple MCQ options extracted: {format_extracted_mcq(gen_mcq_options)}",
+        )
+        if len(orig_mcq_options) == 1:  # Penalize if GT is specific but gen is ambiguous
+            return EvaluateResult(
+                score=0.0,
+                reason="Generated answer is ambiguous (multiple MCQ options) while ground truth is specific.",
+                metrics=metrics,
+            )
+
+    orig_answer_letter = orig_mcq_options[0][1]
+    gen_answer_letter = gen_mcq_options[0][1]
+
+    is_match = orig_answer_letter == gen_answer_letter
+    score = 1.0 if is_match else 0.0
+    reason = f"Match: {is_match}. Gen: '{gen_mcq_options[0][0]}' ({gen_answer_letter}) vs Orig: '{orig_mcq_options[0][0]}' ({orig_answer_letter})"
+
+    metrics["mcq_comparison"] = MetricResult(score=score, is_score_valid=is_match, reason=reason)
+
+    return EvaluateResult(score=score, reason=reason, metrics=metrics)

eval_protocol/rewards/reasoning_steps.py

@@ -0,0 +1,249 @@
+"""
+Reward functions for evaluating reasoning steps.
+
+This module provides reward functions that evaluate whether a model's response
+contains adequate step-by-step reasoning, rewarding structured thinking.
+"""
+
+import re
+from typing import Any, Dict, List, Optional, Pattern, Set, Union
+
+from ..models import EvaluateResult, Message, MetricResult
+from ..typed_interface import reward_function
+
+
+@reward_function
+def reasoning_steps_reward(
+    messages: List[Message],
+    pattern: Optional[str] = None,
+    min_steps: int = 3,
+    max_steps: Optional[int] = None,
+    exclusive_patterns: bool = False,
+    **kwargs: Any,
+) -> EvaluateResult:
+    """
+    Reward function that evaluates step-by-step reasoning in model responses.
+
+    This function checks if the model's response contains indicators of structured
+    reasoning, such as numbered steps, bullet points, or transitional phrases.
+
+    Args:
+        messages: List of conversation messages
+        pattern: Optional custom regex pattern to use for detecting reasoning steps
+        min_steps: Minimum number of steps required for full score
+        max_steps: Optional maximum number of steps (default: None)
+        exclusive_patterns: Whether to use only the custom pattern (True) or
+                           combine it with default patterns (False)
+        **kwargs: Additional arguments
+
+    Returns:
+        EvaluateResult with score based on the number of reasoning steps detected
+    """
+    if not messages or len(messages) == 0:
+        return EvaluateResult(
+            score=0.0,
+            reason="No messages provided",
+            metrics={"reasoning_steps": MetricResult(score=0.0, is_score_valid=False, reason="No messages provided")},
+        )
+
+    response = messages[-1]
+
+    if response.role != "assistant" or not response.content:
+        return EvaluateResult(
+            score=0.0,
+            reason="No assistant response found or response has no content",
+            metrics={
+                "reasoning_steps": MetricResult(
+                    score=0.0,
+                    is_score_valid=False,
+                    reason="Message not from assistant or has no content",
+                )
+            },
+        )
+    text: str = response.content
+
+    # Default patterns for detecting reasoning steps
+    default_patterns = [
+        r"Step\s+\d+[:.]\s+",
+        r"^\s*\d+\.\s+",
+        r"\n\s*\d+\.\s+",
+        r"\n\s*-\s+",
+        r"\n\s*\*\s+",
+        r"\b(?:First|Second|Third|Fourth|Fifth|Next|Then|Finally)[,:]",
+        r"\b(?:Let's|I will|To solve this|To begin)[,:]",
+    ]
+
+    patterns_to_use = []
+    if pattern and exclusive_patterns:
+        patterns_to_use = [pattern]
+    elif pattern:
+        patterns_to_use = [pattern] + default_patterns
+    else:
+        patterns_to_use = default_patterns
+
+    combined_pattern = "|".join(f"(?:{p})" for p in patterns_to_use)
+    matches = re.findall(combined_pattern, text, re.MULTILINE)
+    num_steps = len(matches)
+
+    if num_steps == 0:
+        score = 0.0
+    elif max_steps is not None:
+        score = min(
+            1.0,
+            max(0.0, (num_steps - min_steps + 1) / (max_steps - min_steps + 1)),
+        )
+    else:
+        score = min(1.0, num_steps / min_steps)
+
+    success = num_steps >= min_steps
+    step_metrics = {}
+
+    explicit_steps = len(re.findall(r"Step\s+\d+[:.]\s+", text, re.MULTILINE))
+    if explicit_steps > 0:
+        step_metrics["explicit_steps"] = MetricResult(
+            score=min(1.0, explicit_steps / min_steps),
+            is_score_valid=explicit_steps >= min_steps,
+            reason=f"Found {explicit_steps} explicit steps",
+        )
+
+    numbered_lists = len(re.findall(r"(?:^|\n)\s*\d+\.\s+", text, re.MULTILINE))
+    if numbered_lists > 0:
+        step_metrics["numbered_lists"] = MetricResult(
+            score=min(1.0, numbered_lists / min_steps),
+            is_score_valid=numbered_lists >= min_steps,
+            reason=f"Found {numbered_lists} numbered list items",
+        )
+
+    bullets = len(re.findall(r"(?:^|\n)\s*[-*]\s+", text, re.MULTILINE))
+    if bullets > 0:
+        step_metrics["bullet_points"] = MetricResult(
+            score=min(1.0, bullets / min_steps),
+            is_score_valid=bullets >= min_steps,
+            reason=f"Found {bullets} bullet points",
+        )
+
+    transitions = len(
+        re.findall(
+            r"\b(?:First|Second|Third|Next|Then|Finally)[,:]",
+            text,
+            re.MULTILINE,
+        )
+    )
+    if transitions > 0:
+        step_metrics["transition_phrases"] = MetricResult(
+            score=min(1.0, transitions / min_steps),
+            is_score_valid=transitions >= min_steps,
+            reason=f"Found {transitions} transition phrases",
+        )
+
+    metrics = {
+        "reasoning_steps": MetricResult(
+            score=score,
+            is_score_valid=success,
+            reason=f"Found {num_steps} reasoning steps (minimum required: {min_steps})",
+        ),
+        **step_metrics,
+    }
+
+    reason = f"Detected {num_steps} reasoning steps (required: {min_steps})"
+    if max_steps:
+        reason += f", max: {max_steps}"
+
+    return EvaluateResult(score=score, reason=reason, metrics=metrics)
+
+
+@reward_function
+def sequence_reward(
+    messages: List[Message],
+    sequence_terms: Optional[List[str]] = None,
+    min_matches: int = 3,
+    case_sensitive: bool = False,
+    **kwargs: Any,
+) -> EvaluateResult:
+    """
+    Reward function that evaluates sequential reasoning in model responses.
+
+    This function checks if the model's response follows a specific sequence
+    of reasoning steps or includes a minimum number of required terms in order.
+
+    Args:
+        messages: List of conversation messages
+        sequence_terms: List of terms that should appear in sequence
+        min_matches: Minimum number of sequence terms required for full score
+        case_sensitive: Whether matching should be case-sensitive
+        **kwargs: Additional arguments
+
+    Returns:
+        EvaluateResult with score based on sequence matching
+    """
+    if not messages or len(messages) == 0:
+        return EvaluateResult(
+            score=0.0,
+            reason="No messages provided",
+            metrics={
+                "sequence_reasoning": MetricResult(score=0.0, is_score_valid=False, reason="No messages provided")
+            },
+        )
+
+    response = messages[-1]
+
+    if response.role != "assistant" or not response.content:
+        return EvaluateResult(
+            score=0.0,
+            reason="No assistant response found or response has no content",
+            metrics={
+                "sequence_reasoning": MetricResult(
+                    score=0.0,
+                    is_score_valid=False,
+                    reason="Message not from assistant or has no content",
+                )
+            },
+        )
+    text: str = response.content
+
+    if not sequence_terms:
+        sequence_terms = [
+            "First",
+            "Second",
+            "Third",
+            "Fourth",
+            "Fifth",
+            "Next",
+            "Then",
+            "Finally",
+            "Conclusion",
+        ]
+
+    found_terms = []
+    last_position = -1
+
+    if not case_sensitive:
+        text = text.lower()
+        sequence_terms = [term.lower() for term in sequence_terms]
+
+    for term in sequence_terms:
+        position = text.find(term, last_position + 1)
+        if position > last_position:
+            found_terms.append(term)
+            last_position = position
+
+    num_matches = len(found_terms)
+    score = min(1.0, num_matches / min_matches)
+    success = num_matches >= min_matches
+
+    metrics = {
+        "sequence_reasoning": MetricResult(
+            score=score,
+            is_score_valid=success,
+            reason=f"Found {num_matches} sequential terms (minimum required: {min_matches})",
+        ),
+        "sequential_terms_found": MetricResult(
+            score=score,
+            is_score_valid=success,
+            reason=f"Sequential terms found: {', '.join(found_terms)}",
+        ),
+    }
+
+    reason = f"Detected {num_matches} sequential reasoning terms in order (required: {min_matches})"
+
+    return EvaluateResult(score=score, reason=reason, metrics=metrics)