crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/annotation/chw_care_plan_annotator.py

@@ -0,0 +1,93 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of a
+generated clinical note given unstructured clinical text.
+Your goal is to assess how well the generated response captures the necessary information
+and follows provided instructions in terms of accuracy, structure, and clarity.
+
+The user's request will be provided in these tags:
+<user_request>
+{{QUESTION}}
+</user_request>
+
+The response will be provided in these tags:
+<response>
+{{RESPONSE}}
+</response>
+
+Carefully analyze the <response>.
+For each of the following categories, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Is all the information in the response factually correct?
+
+Structure (1-5)
+- Does the response contain all parts for the provided note generation structure?
+
+Clarity (1-5)
+- Is the response easy to understand for a clinician?
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "structure": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "structure": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
+    "gpt": AnnotatorModelInfo(
+        model_name="openai/gpt-4o-2024-05-13",
+        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
+    ),
+    "llama": AnnotatorModelInfo(
+        model_name="meta/llama-3.3-70b-instruct",
+        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
+    ),
+    "claude": AnnotatorModelInfo(
+        model_name="anthropic/claude-3-7-sonnet-20250219",
+        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
+    ),
+}
+
+
+class CHWCarePlanAnnotator(LLMAsJuryAnnotator):
+    """The CHWCarePlan autograder."""
+
+    name = "chw_care_plan"
+
+    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+        super().__init__(
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=ANNOTATOR_MODELS,
+        )

helm/benchmark/annotation/czech_bank_qa_annotator.py

@@ -0,0 +1,78 @@
+import os
+import sqlite3
+import threading
+from typing import Any, Optional, Tuple
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class CzechBankQAAnnotator(Annotator):
+    """The CzechBankQA autograder.
+
+    MUST BE RUN WITH --num-threads 1 FOR SOME REASON"""
+
+    name = "czech_bank_qa"
+
+    DATABASE_SOURCE_URL = (
+        "https://huggingface.co/datasets/yifanmai/czech_bank_qa/resolve/main/czech_bank.db?download=true"
+    )
+
+    def __init__(self, file_storage_path: str):
+        super().__init__()
+
+        cache_dir = os.path.join(file_storage_path, "data")
+        ensure_directory_exists(cache_dir)
+        file_name = "czech_bank.db"
+        file_path = os.path.join(cache_dir, file_name)
+        ensure_file_downloaded(source_url=CzechBankQAAnnotator.DATABASE_SOURCE_URL, target_path=file_path)
+        database = sqlite3.connect(file_path)
+
+        # csv_files_dir = "/home/yifanmai/oss/helm-scenarios/1999-czech-bank"
+        # # table_name_to_file_name = {
+        # #     "account": "account.csv",
+        # #     "client": "client.csv",
+        # #     "disposition": "disp.csv",
+        # #     "permenant_order": "order.csv",
+        # #     "transaction": "trans.csv",
+        # #     "loan": "loan.csv",
+        # #     "credit_card": "card.csv",
+        # #     "demographic_data": "district.csv"
+        # # }
+        # for file_name in os.listdir(csv_files_dir):
+        #     file_path = os.path.join(csv_files_dir, file_name)
+        #     df = pd.read_csv(file_path)
+        #     table_name = file_name.removesuffix(".csv")
+        #     df.to_sql(table_name, database, if_exists="append", index=False)
+        #     print("Commited to SQL")
+        # # df.to_sql(table_name, conn, if_exists='append', index=False)
+
+        self.database = database
+        self.lock = threading.Lock()
+
+    def get_result(self, query: str) -> Tuple[Optional[str], Optional[str]]:
+        result: Optional[str] = None
+        error: Optional[str] = None
+        try:
+            cursor = self.database.cursor()
+            cursor.execute("PRAGMA query_only = TRUE")
+            cursor.execute(query)
+            result = str(cursor.fetchall())
+            cursor.close()
+        except (sqlite3.DatabaseError, sqlite3.Warning) as e:
+            error = str(e)
+        return (result, error)
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+
+        assert len(request_state.instance.references) == 1
+        gold_query = request_state.instance.references[0].output.text
+        query = request_state.result.completions[0].text
+        query = query.replace("```sql", "").replace("```", "")
+        result, error = self.get_result(query)
+        gold_result, gold_error = self.get_result(gold_query)
+
+        return {"query": query, "result": result, "error": error, "gold_result": gold_result, "gold_error": gold_error}

helm/benchmark/annotation/dischargeme_annotator.py

@@ -0,0 +1,107 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert responsible for evaluating a hospital document.
+The task requires generating either discharge instructions or a brief hospital course based
+on the provided discharge summary and radiology report.
+
+Your goal is to assess whether the generated text is clinically accurate, complete, and clear
+for the intended document type. The evaluation should ensure the document aligns with the
+gold response in terms of accuracy, completeness, and clarity.
+
+The target task of either generating a discharge instruction or brief hospital course along with
+the patient discharge text and radiology report will be provided in these tags:
+<patient_information>
+{{QUESTION}}
+</patient_information>
+
+
+The document will be provided in these tags:
+<response>
+{{RESPONSE}}
+</response>
+
+The gold standard target document (either discharge instructions or a brief hospital course)
+will be provided in these tags:
+<gold_response>
+{{GOLD_RESPONSE}}
+</gold_response>
+
+Carefully analyze the <response> based on the <patient_information> and compare
+it to the <gold_response> when necessary.
+
+For each of the following categories, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent)
+and provide a brief justification for the score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the document provide correct medical information based on the patient's condition and source materials?
+
+Completeness (1-5)
+- Does the document include all important information needed for the specific document type?
+
+Clarity (1-5)
+- Is the document easy to understand for the right audience — patients for discharge
+  instructions or clinicians for the hospital course?
+
+Output Format:
+Generate a valid JSON object with the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
+    "gpt": AnnotatorModelInfo(
+        model_name="openai/gpt-4o-2024-05-13",
+        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
+    ),
+    "llama": AnnotatorModelInfo(
+        model_name="meta/llama-3.3-70b-instruct",
+        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
+    ),
+    "claude": AnnotatorModelInfo(
+        model_name="anthropic/claude-3-7-sonnet-20250219",
+        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
+    ),
+}
+
+
+class DischargeMeAnnotator(LLMAsJuryAnnotator):
+    """The DischargeMe autograder."""
+
+    name = "dischargeme"
+
+    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+        super().__init__(
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=ANNOTATOR_MODELS,
+        )

helm/benchmark/annotation/ehr_sql_annotator.py

@@ -0,0 +1,87 @@
+from typing import Any, List, Optional
+import os
+import re
+import sqlite3
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.common.hierarchical_logger import hwarn
+from helm.benchmark.runner import get_benchmark_output_path
+
+
+class EhrSqlAnnotator(Annotator):
+    """
+    Executes both ground truth and generated SQL queries on the eicu.sqlite database.
+    """
+
+    name = "ehr_sql"
+
+    def annotate(self, request_state: RequestState) -> Any:
+        """Evaluate SQL execution accuracy by running queries against the eicu.sqlite database."""
+
+        databases_root_path = os.path.join(get_benchmark_output_path(), "scenarios", "ehr_sql")
+        database_path = os.path.join(databases_root_path, "eicu.sqlite")
+
+        assert len(request_state.instance.references) == 1
+        ground_truth_sql = request_state.instance.references[0].output.text.strip()
+        ground_truth_result: List[str] = []
+
+        # Execute the ground truth query
+        try:
+            with sqlite3.connect(database_path) as conn:
+                cursor = conn.cursor()
+                cursor.execute(ground_truth_sql)
+                ground_truth_result = cursor.fetchall()
+        except (sqlite3.OperationalError, sqlite3.Warning) as e:
+            hwarn(f"Ground truth SQL failed with error: {e}")
+
+        # If ground truth SQL execution didn't return results, attempt to use extra_data["value"]
+        if not ground_truth_result and request_state.instance.extra_data is not None:
+            if "value" in request_state.instance.extra_data:
+                extra_values = list(request_state.instance.extra_data["value"].values())
+
+                # Try inferring types from the database schema if possible
+                with sqlite3.connect(database_path) as conn:
+                    cursor = conn.cursor()
+                    try:
+                        cursor.execute(ground_truth_sql)
+                        fetched_result = cursor.fetchone()
+                        if fetched_result:
+                            # Convert extra_values to match SQLite's expected types
+                            converted_values = [
+                                type(fetched_result[i])(extra_values[i]) for i in range(len(extra_values))
+                            ]
+                            ground_truth_result = converted_values
+                        else:
+                            # If no rows were fetched, use `extra_values` as-is
+                            ground_truth_result = extra_values
+                    except sqlite3.OperationalError:
+                        # If query fails (syntax error, etc.), just use `extra_values` as-is
+                        ground_truth_result = extra_values
+
+        assert request_state.result is not None
+        assert len(request_state.result.completions) == 1
+        predicted_text = request_state.result.completions[0].text.strip()
+
+        predicted_sql_match = re.search(r"<\s*sql\s*>(.*?)<\/?\s*sql\s*>", predicted_text, re.DOTALL | re.IGNORECASE)
+        predicted_sql = predicted_sql_match.group(1).strip() if predicted_sql_match else predicted_text.strip()
+
+        predicted_result: List[str] = []
+        query_error: Optional[str] = None
+        predicted_sql = predicted_sql.replace("`", "").strip()
+        predicted_sql = re.sub(r"^sql\n", "", predicted_sql, flags=re.MULTILINE)
+        if not predicted_sql:
+            query_error = "No query generated"
+        else:
+            try:
+                with sqlite3.connect(database_path) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute(predicted_sql)
+                    predicted_result = cursor.fetchall()
+            except (sqlite3.OperationalError, sqlite3.Warning) as e:
+                query_error = str(e)
+
+        return {
+            "predicted_result": predicted_result,
+            "ground_truth_result": ground_truth_result,
+            "query_error": query_error,
+        }

helm/benchmark/annotation/helpdesk_call_summarization_annotator.py

@@ -0,0 +1,131 @@
+import re
+from typing import Any, Dict, Optional, Union
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.clients.auto_client import AutoClient
+from helm.common.hierarchical_logger import hwarn
+from helm.common.request import Request
+from helm.proxy.retry import NonRetriableException
+
+
+class AnnotatorResponseParseFailure(NonRetriableException):
+    def __init__(self, response_text: str, **kwargs):
+        self.response_text = response_text
+        super().__init__(kwargs)
+
+
+class HelpdeskCallSummarizationAnnotator(Annotator):
+    """The Helpdesk Call Summarization autograder."""
+
+    name = "helpdesk_call_center_summarization"
+
+    SHORT_NAME_TO_MODEL_INFO: Dict[str, AnnotatorModelInfo] = {
+        "gpt": AnnotatorModelInfo(
+            model_name="openai/gpt-4o-2024-05-13",
+            model_deployment="openai/gpt-4o-2024-05-13",
+        ),
+        "llama": AnnotatorModelInfo(
+            model_name="meta/llama-3.1-405b-instruct-turbo",
+            model_deployment="together/llama-3.1-405b-instruct-turbo",
+        ),
+        "claude": AnnotatorModelInfo(
+            model_name="anthropic/claude-3-5-sonnet-20241022",
+            model_deployment="anthropic/claude-3-5-sonnet-20241022",
+        ),
+    }
+
+    # Template loosely based on these references:
+    # - https://ieeexplore.ieee.org/abstract/document/9946852
+    # - https://arxiv.org/abs/2409.02413v1
+    PROMPT_TEMPLATE = """You are an expert evaluator. Your task is to evaluate the quality of a model-generated summary of a helpdesk call transcript.
+
+The helpdesk call transcript and summary are provided below, delineated with start and end tags:
+
+<call_transcript>
+{{QUESTION}}
+</call_transcript>
+<summary>
+{{PRED}}
+</summary>
+
+Evaluate the summary based on the following criteria:
+
+- Conciseness: A high-quality summary should effectively convey the most important information from the call transcript while keeping the length brief.
+- Relevance: The information presented in the summary should be relevant to the main topic.
+- Coherence: A good summary should have a clear structure and flow of ideas that make it easy to understand and follow.
+- Accuracy: The summary's information should be factually correct and should not contain false or misleading information.
+- Completeness: The summary should a fair description of the main problems (concerns) and resolutions based on the original call transcripts.
+
+Think step by step, then score the summary. Your reasoning should be less than 200 words. The score should be a single number between 1 to 10 inclusive.
+
+Please respond with your output and reasoning in the following format, your reasoning within <reasoning></reasoning> tags and your score within <score></score> tags, without any other output:
+
+<reasoning>INSERT_YOUR_REASONING_HERE</reasoning>
+<score>INSERT_YOUR_SCORE_HERE</score>
+"""  # noqa: E501
+
+    PATTERN = r"^\s*reason:(.*)##(.*)"
+
+    def __init__(self, auto_client: AutoClient):
+        self._auto_client = auto_client
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        prediction_text = request_state.result.completions[0].text
+
+        question_text = request_state.instance.input.text
+
+        annotator_prompt = self.PROMPT_TEMPLATE.replace("{{QUESTION}}", question_text).replace(
+            "{{PRED}}", prediction_text
+        )
+        annotations: Dict[str, Union[Optional[str], Optional[float]]] = {"prompt_text": annotator_prompt}
+        for annotator_name, annotator_model_info in self.SHORT_NAME_TO_MODEL_INFO.items():
+            annotator_request = Request(
+                model=annotator_model_info.model_name,
+                model_deployment=annotator_model_info.model_deployment,
+                prompt=annotator_prompt,
+                temperature=0.0,
+                max_tokens=512,
+            )
+            annotator_response = self._auto_client.make_request(annotator_request)
+            if not annotator_response.success:
+                raise Exception(f"Annotation request failed: {annotator_response.error}")
+            assert len(annotator_response.completions) == 1
+            annotator_response_text = annotator_response.completions[0].text
+            # fuzzy match regex check, allows for different casing, or forgetting / in end tag
+            reasoning_match = re.search(
+                r"<\s*reasoning\s*>(.*?)<\/?\s*reasoning\s*>", annotator_response_text, re.DOTALL | re.IGNORECASE
+            )
+            score_match = re.search(
+                r"<\s*score\s*>(.*?)<\/?\s*score\s*>", annotator_response_text, re.DOTALL | re.IGNORECASE
+            )
+            reasoning: Optional[str] = None
+            score: Optional[float] = None
+            if reasoning_match:
+                reasoning = reasoning_match.group(1).strip()
+            else:
+                hwarn(
+                    "HelpdeskCallSummarizationAnnotator could not get Reasoning from annotation from "
+                    f"{annotator_model_info.model_name}: {annotator_response_text}"
+                )
+
+            if score_match:
+                try:
+                    score = float(score_match.group(1).strip())
+                except ValueError:
+                    hwarn(
+                        "HelpdeskCallSummarizationAnnotator could not parse Score from annotation from "
+                        f"{annotator_model_info.model_name}: {annotator_response_text}"
+                    )
+            else:
+                hwarn(
+                    "HelpdeskCallSummarizationAnnotator could not get Score from annotation from "
+                    f"{annotator_model_info.model_name}: {annotator_response_text}"
+                )
+
+            annotations[f"{annotator_name}_reasoning"] = reasoning
+            annotations[f"{annotator_name}_score"] = score
+        return annotations

helm/benchmark/annotation/image2struct/image_compiler_annotator.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from threading import Lock
 from typing import Any, Dict, List, Tuple, Callable
 
 from dacite import from_dict
@@ -17,6 +18,9 @@ except ModuleNotFoundError as e:
     handle_module_not_found_error(e, suggestions=["images"])
 
 
+_compilation_lock = Lock()
+
+
 def retry_if_compilation_failed(result: Dict[str, Any]) -> bool:
     """Retries when the compilation fails."""
     return "unknown_error" in result
@@ -78,7 +82,8 @@ class ImageCompilerAnnotator(Annotator, ABC):
                 except Exception as e:
                     return {"unknown_error": str(e)}
 
-            raw_response = compile()
+            with _compilation_lock:
+                raw_response = compile()
             response = {**raw_response}
             if "media_object" in response:
                 response["media_object"] = from_dict(MediaObject, response["media_object"])

helm/benchmark/annotation/live_qa_annotator.py

@@ -5,7 +5,7 @@ from typing import Any
 from helm.common.general import ensure_directory_exists
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.annotation.annotator import Annotator
-from helm.benchmark.annotation.score_util import score_with_reasoning
+from helm.benchmark.annotation.model_as_judge import score_with_reasoning
 from helm.clients.auto_client import AutoClient
 
 PROMPT_TEMPLATE = """You are a medical judge will be given a user request and a model response to that request.

helm/benchmark/annotation/med_dialog_annotator.py

@@ -0,0 +1,99 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of medical dialogue summaries.
+
+Your goal is to assess how well the summary captures the patient-doctor conversation,
+and how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The patient-doctor conversation will be provided in these tags:
+<conversation>
+{{QUESTION}}
+</conversation>
+
+The response will be provided in these tags:
+<response>
+{{RESPONSE}}
+</response>
+
+The reference response will be provided in these tags:
+<gold_response>
+{{GOLD_RESPONSE}}
+</gold_response>
+
+Carefully review the <response> and compare it to the <gold_response> when needed.
+
+Please evaluate the generated summary on a scale of 1-5 (1 = poor, 5 = excellent)
+for each of these three key dimensions:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the summary correctly capture the main medical issue and clinical details from the conversation?
+
+Completeness (1-5)
+- Does the summary include all important medical information from the conversation?
+
+Clarity (1-5)
+- Is the summary easy to understand for clinical use?
+
+Output Format:
+Output your evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
+    "gpt": AnnotatorModelInfo(
+        model_name="openai/gpt-4o-2024-05-13",
+        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
+    ),
+    "llama": AnnotatorModelInfo(
+        model_name="meta/llama-3.3-70b-instruct",
+        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
+    ),
+    "claude": AnnotatorModelInfo(
+        model_name="anthropic/claude-3-7-sonnet-20250219",
+        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
+    ),
+}
+
+
+class MedDialogAnnotator(LLMAsJuryAnnotator):
+    """The MedDialog autograder."""
+
+    name = "med_dialog"
+
+    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+        super().__init__(
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=ANNOTATOR_MODELS,
+        )