crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/annotation/image2struct/webpage_compiler_annotator.py

@@ -0,0 +1,132 @@
+from typing import List, Tuple, Optional, Dict, Any
+import json
+import os
+import shutil
+import threading
+
+from helm.benchmark.annotation.image2struct.image_compiler_annotator import ImageCompilerAnnotator, CompilationError
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.scenarios.vision_language.image2struct.webpage.driver import ScreenshotOptions
+from helm.benchmark.scenarios.vision_language.image2struct.webpage.utils import convert_html_to_text
+from helm.benchmark.scenarios.vision_language.image2struct.webpage_scenario import serve_and_take_screenshot
+from helm.benchmark.scenarios.scenario import ASSET_NAME_TAG, ASSET_PATH_TAG
+from helm.common.general import ensure_directory_exists
+from helm.common.cache import CacheConfig
+
+try:
+    from PIL import Image
+    from html2text import HTML2Text
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["image2struct"])
+
+
+class WebpageCompilerAnnotator(ImageCompilerAnnotator):
+    """Annotator that compiles the text completions into a webpage
+    And takes a screenshot of the webpage."""
+
+    name: str = "webpage_compiler"
+
+    # Delimiters for the code block
+    DELIMITERS: List[Tuple[str, str]] = [
+        ("```json", "```"),
+        ("```", "```"),
+    ]
+
+    def __init__(self, cache_config: CacheConfig, file_storage_path: str):
+        super().__init__(cache_config, file_storage_path)
+        self._html2text = HTML2Text()
+        self._html2text.ignore_links = True
+
+    def postprocess_infos(self, infos: Dict[str, Any]) -> Dict[str, Any]:
+        """Postprocess the infos."""
+        annotations = super().postprocess_infos(infos)
+        assert "html" in annotations, "The html field should be present in the infos"
+        annotations["text"] = convert_html_to_text(self._html2text, infos["html"])
+        return annotations
+
+    def compile_completion_into_image(
+        self, request_state: RequestState, completion_text: str
+    ) -> Tuple[Image.Image, Dict[str, Any]]:
+        """Given a completion, parse the code and compile it into an image and return the image and the infos."""
+        # Create a temporary directory to store the files
+        cache_config: CacheConfig = self._cache.config
+        repo_path: str = "prod_env/tmp"
+        if hasattr(cache_config, "path"):
+            repo_path = os.path.join(os.path.dirname(cache_config.path), "tmp")
+        # Make the repo path thread safe by adding the thread id
+        repo_path = f"{repo_path}_{threading.get_ident()}"
+        ensure_directory_exists(repo_path)
+
+        # Check for code block delimiters
+        # After this completion should be a valid json object
+        for start, end in self.DELIMITERS:
+            if start in completion_text and end in completion_text[completion_text.index(start) + len(start) :]:
+                start_index = completion_text.index(start) + len(start)
+                end_index = completion_text.index(end, start_index)
+                completion_text = completion_text[start_index:end_index]
+                break
+
+        # Parse code into json object
+        structure: dict
+        try:
+            structure = json.loads(completion_text)
+        except json.JSONDecodeError as e:
+            raise CompilationError(f"Failed to parse the completion as a JSON object: {e}") from e
+
+        # Copy the assets
+        assets_paths: List[str] = []
+        assets_names: List[str] = []
+        for reference in request_state.instance.references:
+            if ASSET_PATH_TAG in reference.tags:
+                assert reference.output.multimedia_content is not None
+                for media_object in reference.output.multimedia_content.media_objects:
+                    assert media_object.is_local_file
+                    assert media_object.is_type("image")
+                    assert type(media_object.location) == str
+                    assets_paths.append(media_object.location)
+            if ASSET_NAME_TAG in reference.tags:
+                assert reference.output.multimedia_content is not None
+                for media_object in reference.output.multimedia_content.media_objects:
+                    assert media_object.is_type("text")
+                    assert type(media_object.text) == str
+                    assets_names.append(media_object.text)
+        assert len(assets_paths) == len(assets_names)
+        for asset_path, asset_name in zip(assets_paths, assets_names):
+            dest_path: str = os.path.join(repo_path, asset_name)
+            # Make sure the parent directory exists
+            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+            shutil.copyfile(asset_path, dest_path)
+            # os.symlink(asset_path, dest_path)
+
+        # Create each file in a temporary directory
+        if not isinstance(structure, list):
+            raise CompilationError("The completion should be a list of files")
+        for item in structure:
+            filename: Optional[str] = item.get("filename")
+            content: Optional[str] = item.get("content")
+            if filename is None or content is None:
+                raise CompilationError("Each file should have a valid filename and content")
+            # Create parent directories if they do not exist
+            if filename in assets_names:
+                # Some models will include assets in their response like this:
+                # {
+                #     "filename": "chmber.jpg",
+                #     "content": "The content of the chmber.jpg file is a binary image and cannot be displayed as text."
+                # }
+                # In this case, we skip the file creation
+                continue
+            parent_dir = os.path.join(repo_path, os.path.dirname(filename))
+            os.makedirs(parent_dir, exist_ok=True)
+            with open(os.path.join(repo_path, filename), "w") as f:
+                f.write(content)
+
+        # Save the screenshot, loads the image and remove the file
+        destination_path: str = os.path.join(repo_path, "output.png")
+        infos: Dict[str, Any] = serve_and_take_screenshot(repo_path, destination_path, ScreenshotOptions())
+        image: Image.Image = Image.open(destination_path)
+
+        # Delete the repository
+        shutil.rmtree(repo_path, ignore_errors=True)
+
+        return image, infos

helm/benchmark/annotation/live_qa_annotator.py

@@ -0,0 +1,76 @@
+import os
+import re
+from typing import Any
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.benchmark.annotation.model_as_judge import score_with_reasoning
+from helm.clients.auto_client import AutoClient
+
+PROMPT_TEMPLATE = """You are a medical judge will be given a user request and a model response to that request.
+Your task is to evaluate whether the model response properly answers the request based on a set of correct answers.
+The user's request will be provided in these tags:
+<user_request>
+{{QUESTION}}
+</user_request>
+The model's response will be provided in these tags:
+<model_response>
+{{ANSWER}}
+</model_response>
+Some potential correct responses will be provided in these tags:
+<correct_responses>
+{{CORRECT_RESPONSES}}
+</correct_responses>
+Carefully analyze the <model_response> contents.
+Determine if the model is correct with respect to one or more of the <correct_responses> on a range from 0 to 1.
+# 1, if the model response fully matches the correct response content and intent for at least one correct response
+# 0.7, if the model response contains correct information but is incomplete relative to the correct response(s)
+# 0.3, if the model response contains some correct information but is incomplete and/or incorrect
+# 0, if the model response is completely unrelated or incorrect relative to the correct response(s)
+Please output your one-sentence concise reasoning within the "reasoning" tags and your score within the "score" tags.
+Your reasoning should be less than 20 tokens. The score should be a single number with no other output.
+Only output a tag-delimited object with the following format:
+<reasoning>
+INSERT_YOUR_REASONING_HERE
+</reasoning>
+<score>
+INSERT_YOUR_SCORE_HERE
+</score>
+"""
+
+
+class LiveQAAnnotator(Annotator):
+    """The LiveQA autograder."""
+
+    name = "live_qa"
+
+    def __init__(self, auto_client: AutoClient, file_storage_path: str):
+        self._auto_client = auto_client
+        cache_dir = os.path.join(file_storage_path, "data")
+        ensure_directory_exists(cache_dir)
+        # Regex pattern is lenient to allow for typos e.g. extra whitespace
+        self._pattern = re.compile(r"##\s*short_reasoning\s*:(.*)##\s*the_score\s*:(.*)", re.DOTALL)
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        model_input_text = request_state.request.prompt
+        model_output_text = request_state.result.completions[0].text
+        if not model_output_text.strip():
+            return {"prompt_text": "", "reasoning": "BLOCKED_REQUEST_OR_EMPTY_RESPONSE", "score": 0.0}
+        correct_responses = ""
+        for i, reference in enumerate(request_state.instance.references):
+            correct_responses += "Correct Response #" + str(i) + ": " + reference.output.text + "\n"
+        annotator_prompt = (
+            PROMPT_TEMPLATE.strip()
+            .replace("{{QUESTION}}", model_input_text)
+            .replace("{{ANSWER}}", model_output_text)
+            .replace("{{CORRECT_RESPONSES}}", correct_responses)
+        )
+        results = score_with_reasoning(
+            self._auto_client, annotator_prompt, "openai/gpt-4o-2024-05-13", "openai/gpt-4o-2024-05-13"
+        )
+        reasoning = results.get("reasoning")
+        score = results.get("score")
+        return {"prompt_text": annotator_prompt, "reasoning": reasoning, "score": score}

helm/benchmark/annotation/med_dialog_annotator.py

@@ -0,0 +1,88 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of medical dialogue summaries.
+
+Your goal is to assess how well the summary captures the patient-doctor conversation,
+and how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The patient-doctor conversation will be provided in these tags:
+<conversation>
+{QUESTION}
+</conversation>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The reference response will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully review the <response> and compare it to the <gold_response> when needed.
+
+Please evaluate the generated summary on a scale of 1-5 (1 = poor, 5 = excellent)
+for each of these three key dimensions:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the summary correctly capture the main medical issue and clinical details from the conversation?
+
+Completeness (1-5)
+- Does the summary include all important medical information from the conversation?
+
+Clarity (1-5)
+- Is the summary easy to understand for clinical use?
+
+Output Format:
+Output your evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MedDialogAnnotator(LLMAsJuryAnnotator):
+    """The MedDialog autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="med_dialog",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/medalign_annotator.py

@@ -0,0 +1,89 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert responsible for evaluating the quality of a
+response to a clinical instruction, given an associated Electronic Health Record (EHR).
+
+Your goal is to assess how well the response interprets the EHR, follows clinical best practices,
+and aligns with the gold response in terms of accuracy, completeness, and clarity.
+
+The instruction and EHR pair will be provided in these tags:
+<user_request>
+{QUESTION}
+</user_request>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The gold response (reference answer) will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully review the <response> based on the <user_request> and compare it to
+the <gold_response> when needed.
+For each category below, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent)
+and provide a brief explanation for your score:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the response provide correct medical information based on the EHR and clinical guidelines?
+
+Completeness (1-5)
+- Does the response include all important patient information and address the clinical question?
+
+Clarity (1-5)
+- Is the response written clearly so clinicians can easily understand it?
+
+Output Format:
+Generate a valid JSON object with your evaluation:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MedalignAnnotator(LLMAsJuryAnnotator):
+    """The Medalign autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="medalign",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/medi_qa_annotator.py

@@ -0,0 +1,87 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of a
+response to a consumer health question.
+Your goal is to assess how well the generated response answers the medical question,
+and how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The question will be provided in these tags:
+<question>
+{QUESTION}
+</question>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The reference answer will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully analyze the <response> compared to the <gold_response> and the original <question>.
+For each of the following criteria, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Is the medical information correct?
+
+Completeness (1-5)
+- Does the response fully answer the question and include important medical details?
+
+Clarity (1-5)
+- Is the response easy for consumers to understand?
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MediQAAnnotator(LLMAsJuryAnnotator):
+    """The MediQA autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="medi_qa",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/medication_qa_annotator.py

@@ -0,0 +1,86 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating responses to consumer medication questions.
+
+Your goal is to assess how well the response captures the information asked,
+and how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The question provided in these tags:
+<medication_question>
+{QUESTION}
+</medication_question>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The reference response will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully review the <response> and compare it to the <gold_response> when needed.
+Please evaluate the response on a scale of 1-5 (1 = poor, 5 = excellent) for each of these three key dimensions:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Is the information about the medication factually correct?
+
+Completeness (1-5)
+- Does the response fully answer the question, covering all important points?
+
+Clarity (1-5)
+- Is the response easy for a consumer to understand?
+
+Output Format:
+Output your evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MedicationQAAnnotator(LLMAsJuryAnnotator):
+    """The MedicationQA autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="medication_qa",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/mental_health_annotator.py

@@ -0,0 +1,87 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a mental health expert tasked with evaluating the quality
+of a counselor response in a therapy conversation.
+Your goal is to assess how well the generated response demonstrates therapeutic skills,
+and how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The conversation history and question will be provided in these tags:
+<conversation>
+{QUESTION}
+</conversation>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The reference response will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully analyze the <response> compared to the <gold_response> and the original <conversation>.
+For each of the following criteria, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the response use appropriate therapeutic techniques and show understanding of the client's concerns?
+
+Completeness (1-5)
+- Does the response address the client's key concerns and include important context from the conversation?
+
+Clarity (1-5)
+- Is the response easy to understand and communicated in an empathetic, therapeutic way?
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class MentalHealthAnnotator(LLMAsJuryAnnotator):
+    """The MentalHealth autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="mental_health",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )