crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/annotation/mimic_bhc_annotator.py

@@ -0,0 +1,89 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are tasked with evaluating the quality of the generated brief hospital
+course based on the provided clinical note.
+Your goal is to assess how well the brief hospital course captures all the clinical details and
+compares to the gold response in terms of accuracy, completeness, and clarity.
+
+
+The user's request will be provided in these tags:
+<user_request>
+{QUESTION}
+</user_request>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+A potential correct response will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully analyze the <response>. For each of the following categories,
+rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Your evaluation should focus on the following criteria:
+Accuracy (1-5)
+- Does the brief hospital course correctly reflect the key details from the clinical note?
+
+Completeness (1-5)
+- Does the brief hospital course include all important details and address the clinical scenario?
+
+Clarity (1-5)
+-Is the brief hospital course easy for clinicians to understand?
+
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MIMICBHCAnnotator(LLMAsJuryAnnotator):
+    """The MIMICBHC autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="mimic_bhc",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/mimic_rrs_annotator.py

@@ -0,0 +1,89 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are tasked with evaluating the quality of the generated impression section
+of a radiology report based on the provided findings.
+Your goal is to assess how well the impression section captures the all the clinical findings and
+how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The user's request will be provided in these tags:
+<user_request>
+{QUESTION}
+</user_request>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+Some potential correct responses will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully analyze the <response>.
+For each of the following categories, rate the Response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Your evaluation should focus on the following criteria:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the impression correctly reflect the key findings from the radiology report?
+
+Completeness (1-5)
+- Does the impression include all important findings and address the clinical question?
+
+Clarity (1-5)
+- Is the impression easy for referring clinicians to understand?
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MIMICRRSAnnotator(LLMAsJuryAnnotator):
+    """The MIMICRRS autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="mimic_rrs",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/model_as_judge.py

@@ -0,0 +1,309 @@
+import json
+import re
+from dataclasses import dataclass
+from string import Template
+from typing import Dict, Optional, TypedDict, Union, Callable, Any, Set
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.clients.auto_client import AutoClient
+from helm.common.hierarchical_logger import hlog, hwarn
+from helm.common.request import Request
+
+
+class AnnotatorResponseParseFailure(Exception):
+    def __init__(self, response_text: str, **kwargs):
+        self.response_text = response_text
+        super().__init__(kwargs)
+
+
+@dataclass
+class AnnotatorModelInfo:
+    model_name: str
+    model_deployment: str
+
+
+def score_with_reasoning_with_gpt_and_llama(
+    auto_client: AutoClient,
+    annotator_prompt: str,
+) -> Dict[str, Optional[Union[str, float]]]:
+    """EXPERIMENTAL: DO NOT USE IN PRODUCTION
+
+    Score using GPT-4o and Llama 3.1 for safety scenarios in HELM Safety."""
+    # TODO: Make this configurable
+    SHORT_NAME_TO_MODEL_INFO: Dict[str, AnnotatorModelInfo] = {
+        "gpt": AnnotatorModelInfo(model_name="openai/gpt-4o-2024-05-13", model_deployment="openai/gpt-4o-2024-05-13"),
+        "llama": AnnotatorModelInfo(
+            model_name="meta/llama-3.1-405b-instruct-turbo", model_deployment="together/llama-3.1-405b-instruct-turbo"
+        ),
+    }
+    result: Dict[str, Optional[Union[str, float]]] = {"prompt_text": annotator_prompt}
+    for short_name, model_info in SHORT_NAME_TO_MODEL_INFO.items():
+        try:
+            score_and_reasoning = score_with_reasoning(
+                auto_client,
+                annotator_prompt,
+                annotator_model=model_info.model_name,
+                annotator_model_deployment=model_info.model_deployment,
+            )
+            result[f"{short_name}_score"] = score_and_reasoning["score"]
+            result[f"{short_name}_reasoning"] = score_and_reasoning["reasoning"]
+        except AnnotatorResponseParseFailure as e:
+            result[f"{short_name}_score"] = None
+            result[f"{short_name}_reasoning"] = None
+            result[f"{short_name}_response_text"] = e.response_text
+    return result
+
+
+class ScoreAndReasoning(TypedDict):
+    score: float
+    reasoning: str
+
+
+def score_with_reasoning(
+    auto_client: AutoClient,
+    annotator_prompt: str,
+    annotator_model: str,
+    annotator_model_deployment: str,
+) -> ScoreAndReasoning:
+    annotator_request = Request(
+        model=annotator_model,
+        model_deployment=annotator_model_deployment,
+        prompt=annotator_prompt,
+        temperature=0.0,
+        max_tokens=256,
+    )
+    annotator_response = auto_client.make_request(annotator_request)
+    if not annotator_response.success:
+        raise Exception(f"Annotation request failed: {annotator_response.error}")
+    assert len(annotator_response.completions) == 1
+    annotator_response_text = annotator_response.completions[0].text
+
+    # fuzzy match regex check, allows for different casing, or forgetting / in end tag
+    reasoning_match = re.search(
+        r"<\s*reasoning\s*>(.*?)<\/?\s*reasoning\s*>", annotator_response_text, re.DOTALL | re.IGNORECASE
+    )
+    score_match = re.search(r"<\s*score\s*>(.*?)<\/?\s*score\s*>", annotator_response_text, re.DOTALL | re.IGNORECASE)
+    if not reasoning_match or not score_match:
+        raise AnnotatorResponseParseFailure(
+            message=f"Could not parse markup in raw response: '{annotator_response_text}'",
+            response_text=annotator_response_text,
+        )
+    reasoning = reasoning_match.group(1).strip()
+    try:
+        score = float(score_match.group(1).strip())
+    except ValueError:
+        raise AnnotatorResponseParseFailure(
+            message=f"Could not parse score as float from raw request: '{annotator_response_text}'",
+            response_text=annotator_response_text,
+        )
+
+    return {"reasoning": reasoning, "score": score}
+
+
+class LLMAsJuryAnnotator(Annotator):
+    """
+    A flexible LLM-based annotator that can be configured for different annotation scenarios.
+
+    This annotator supports:
+    - Custom prompt templates
+    - Multiple evaluation models
+    - Configurable evaluation criteria
+    - Robust error handling
+    """
+
+    def __init__(
+        self,
+        name: str,
+        auto_client: AutoClient,
+        prompt_template: str,
+        annotation_criteria: Dict[str, Set[str]],
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        preprocessor: Optional[Callable[[str], str]] = None,
+    ):
+        """
+        Initialize the LLMAsJuryAnnotator.
+
+        :param auto_client: Client for making API requests
+        :param prompt_template: Template for generating prompts
+        :param annotation_criteria: Dictionary defining expected annotation structure
+        :param annotator_models: Dictionary of models to use for annotation
+        :param preprocessor: Optional function to preprocess model responses
+        """
+        self.name = name
+        self._auto_client = auto_client
+        self._prompt_template = prompt_template
+        self._annotation_criteria = annotation_criteria
+        self._annotator_models = annotator_models
+        self._preprocessor = preprocessor or self._sanitize_model_response
+
+    def _sanitize_model_response(self, model_response: str) -> str:
+        """
+        Sanitize the model response to extract JSON.
+
+        :param model_response: Raw model response
+        :return: Extracted JSON string
+        """
+        json_match = re.search(r"\{.*\}", model_response, re.DOTALL)
+        return json_match.group(0) if json_match else model_response
+
+    def _interpolate_prompt(
+        self, request_state: RequestState, custom_replacements: Optional[Dict[str, str]] = None
+    ) -> str:
+        """Interpolate prompt templates safely, supporting {{QUESTION}}-style files."""
+        # Build required/optional fields
+        replacements: Dict[str, str] = {
+            "QUESTION": request_state.instance.input.text,
+            "RESPONSE": (
+                request_state.result.completions[0].text
+                if request_state.result and request_state.result.completions
+                else ""
+            ),
+            # GOLD is optional; keep empty if not present
+            "GOLD_RESPONSE": (
+                request_state.instance.references[0].output.text
+                if getattr(request_state.instance, "references", None)
+                else ""
+            ),
+        }
+        if custom_replacements:
+            replacements.update(custom_replacements)
+
+        tmpl_text = self._prompt_template
+
+        tmpl_text = (
+            tmpl_text.replace("{QUESTION}", "$QUESTION")
+            .replace("{RESPONSE}", "$RESPONSE")
+            .replace("{GOLD_RESPONSE}", "$GOLD_RESPONSE")
+        )
+
+        return Template(tmpl_text).substitute(replacements)
+
+    def _validate_annotation(self, annotator_criteria: Dict[str, Any], annotator_name: str) -> bool:
+        """
+        Validate the annotation meets expected criteria.
+
+        :param annotator_criteria: Annotation dictionary to validate
+        :param annotator_name: Name of the annotator model
+        :return: Whether the annotation is valid
+        """
+        for key, value in self._annotation_criteria.items():
+            if key not in annotator_criteria:
+                hwarn(f"Annotator did not find the expected key " f"'{key}' in the response from {annotator_name}.")
+                return False
+
+            for subkey in value:
+                if subkey not in annotator_criteria[key]:
+                    hwarn(
+                        f"Annotator did not find the expected subkey "
+                        f"'{subkey}' in the response from {annotator_name}."
+                    )
+                    return False
+        return True
+
+    def annotate(self, request_state: RequestState) -> Dict[str, Any]:
+        """
+        Annotate the request state using multiple LLM models.
+
+        :param request_state: The request state to annotate
+        :return: Dictionary of annotations from different models
+        """
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+
+        # Check for empty model output
+        model_output_text = request_state.result.completions[0].text
+        if not model_output_text.strip():
+            hwarn("Annotator skipped sending requests because the model response was empty")
+            return {
+                "prompt_text": None,
+                "empty_output_equivalence_judgement": False,
+            }
+
+        # Prepare prompt
+        annotator_prompt = self._interpolate_prompt(request_state)
+        annotations: Dict[str, Union[Optional[str], Optional[bool], Dict[str, Any]]] = {"prompt_text": annotator_prompt}
+
+        # Track failed annotations for each model
+        failed_counts: Dict[str, int] = {name: 0 for name in self._annotator_models}
+
+        # Annotate using multiple models
+        for annotator_name, annotator_model_info in self._annotator_models.items():
+            try:
+                annotator_criteria = self._annotate_with_model(annotator_prompt, annotator_model_info, annotator_name)
+
+                if annotator_criteria is not None:
+                    annotations[annotator_name] = annotator_criteria
+                else:
+                    failed_counts[annotator_name] += 1
+
+            except Exception as e:
+                hlog(f"ERROR annotating with {annotator_name}: {e}")
+                failed_counts[annotator_name] += 1
+
+        hlog(f"Failed model annotations: {failed_counts}")
+        return annotations
+
+    def _annotate_with_model(
+        self, prompt: str, model_info: AnnotatorModelInfo, annotator_name: str
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Annotate using a specific model with enhanced JSON parsing.
+
+        :param prompt: Interpolated prompt
+        :param model_info: Model information
+        :param annotator_name: Name of the annotator
+        :return: Annotation criteria or None if failed
+        """
+        annotator_request = Request(
+            model=model_info.model_name,
+            model_deployment=model_info.model_deployment,
+            prompt=prompt,
+            temperature=0.0,
+            max_tokens=4096,
+        )
+
+        annotator_response = self._auto_client.make_request(annotator_request)
+
+        if not annotator_response.success:
+            hwarn(f"Got an error response from {model_info.model_name}: " f"{annotator_response.error}")
+            return None
+
+        try:
+            annotator_output = annotator_response.completions[0].text
+            annotator_output = self._preprocessor(annotator_output)
+
+            try:
+                annotator_criteria = json.loads(annotator_output)
+            except json.JSONDecodeError as e:
+                if e.msg == "Expecting ',' delimiter":
+                    # Attempt to fix incomplete JSON by adding a closing brace
+                    annotator_output = annotator_output + "}"
+                    try:
+                        annotator_criteria = json.loads(annotator_output)
+                    except Exception as ex:
+                        hwarn(
+                            f"Error parsing response from {model_info.model_name} "
+                            f"after adding closing brace: {ex}. "
+                            f"Model output: {annotator_output}"
+                        )
+                        return None
+                else:
+                    # For other JSON decoding errors
+                    hwarn(
+                        f"JSON decoding error from {model_info.model_name}: {e}. " f"Model output: {annotator_output}"
+                    )
+                    return None
+
+            # Validate annotation structure
+            if not self._validate_annotation(annotator_criteria, annotator_name):
+                return None
+
+            return annotator_criteria
+
+        except Exception as e:
+            hwarn(
+                f"Unexpected error processing response from {model_info.model_name}: {e}. "
+                f"Model output: {annotator_output}"
+            )
+            return None

helm/benchmark/annotation/mtsamples_procedures_annotator.py

@@ -0,0 +1,87 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of a
+generated response of a clinical scenario.
+Your goal is to assess how well the response addresses the patient case, follows clinical best practices,
+and compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The user's request will be provided in these tags:
+<user_request>
+{QUESTION}
+</user_request>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+Some potential correct responses will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully analyze the <response>.
+For each of the following categories, rate the Response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the response provide correct medical advice based on clinical guidelines and the patient case?
+
+Completeness (1-5)
+- Does the response cover all important aspects of care for this clinical scenario?
+
+Clarity (1-5)
+- Is the response written clearly and organized in a way that clinicians can easily follow?
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MTSamplesProceduresAnnotator(LLMAsJuryAnnotator):
+    """The MTSamplesProcedures autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="mtsamples_procedures",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/mtsamples_replicate_annotator.py

@@ -0,0 +1,90 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert responsible for evaluating a proposed treatment plan
+based on provided patient information.
+
+Your goal is to assess whether the treatment plan uses the patient's medical history,
+medications, symptoms, and other relevant details appropriately, follows clinical best practices,
+and aligns with the gold standard response in accuracy, completeness, and clarity.
+
+The patient's information will be provided in these tags:
+<patient_information>
+{QUESTION}
+</patient_information>
+
+The proposed treatment plan will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The gold standard treatment plan will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully review the <response> based on the <patient_information> and compare it to the <gold_response> when needed.
+
+For each category below, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent) and
+provide a brief explanation for your score:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the treatment plan provide correct medical advice based on clinical guidelines?
+
+Completeness (1-5)
+Does the treatment plan include all important medical details?
+
+Clarity (1-5)
+- Is the treatment plan written clearly so clinicians can easily understand it?
+
+Output Format:
+Generate a valid JSON object with the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MTSamplesReplicateAnnotator(LLMAsJuryAnnotator):
+    """The MTSamplesReplicate autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="mtsamples_replicate",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )