crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/annotation/bigcodebench_annotator.py

@@ -0,0 +1,108 @@
+from typing import Any, List, Dict, Optional
+import ast
+import json
+import tempfile
+import traceback
+
+from gradio_client import Client, handle_file
+from retrying import retry
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.common.hierarchical_logger import hlog, hwarn
+
+
+# Based on https://github.com/bigcode-project/bigcodebench/blob/0331489b29cbf2653b4669597ef431e158882aab/bigcodebench/syncheck.py#L14  # noqa: E501
+# Licensed under Apache 2.0
+def syntax_check(code, verbose=False):
+    try:
+        ast.parse(code)
+        return True
+    except (SyntaxError, MemoryError):
+        if verbose:
+            traceback.print_exc()
+        return False
+
+
+# Based on https://github.com/bigcode-project/bigcodebench/blob/0331489b29cbf2653b4669597ef431e158882aab/bigcodebench/sanitize.py#L30  # noqa: E501
+# Licensed under Apache 2.0
+def code_extract(text: str) -> str:
+    lines = text.split("\n")
+    longest_line_pair = (0, 0)
+    longest_so_far = 0
+
+    for i in range(len(lines)):
+        for j in range(i + 1, len(lines)):
+            current_lines = "\n".join(lines[i : j + 1])
+            if syntax_check(current_lines):
+                current_length = sum(1 for line in lines[i : j + 1] if line.strip())
+                if current_length > longest_so_far:
+                    longest_so_far = current_length
+                    longest_line_pair = (i, j)
+
+    return "\n".join(lines[longest_line_pair[0] : longest_line_pair[1] + 1])
+
+
+class BigCodeBenchAnnotator(Annotator):
+    """The BigCodeBench autograder."""
+
+    name = "bigcodebench"
+
+    DEFAULT_URL = "https://bigcode-bigcodebench-evaluator.hf.space/"
+    SPLIT = "instruct"
+    SUBSET = "full"
+    PASS_K = "1"
+    DATASET_SIZE = 1140
+
+    def __init__(self, api_key: Optional[str], endpoint: Optional[str]):
+        self.use_global_metric = True
+        if api_key and endpoint:
+            hlog(f"BigCodeBenchAnnotator will use the configured endpoint {endpoint}")
+            self.client = Client(endpoint, hf_token=api_key)
+        else:
+            hwarn(
+                f"BigCodeBenchAnnotator will use the default public evaluator endpoint {self.DEFAULT_URL} - "
+                "set bigcodebenchApiKey and bigcodebenchEndpoint in credentials.conf to use a cloned evaluator instead"
+            )
+            self.client = Client(self.DEFAULT_URL)
+
+    def annotate(self, request_state: RequestState) -> Any:
+        raise NotImplementedError("annotate() is not supported; use annotate_all() instead")
+
+    @retry(stop_max_attempt_number=3, wait_fixed=4000)
+    def send_request_to_gradio_evaluator(self, filename: str, task_ids: List[str]):
+        if len(task_ids) == self.DATASET_SIZE:
+            selective_evaluate = ""
+        else:
+            selective_evaluate = ",".join([task_id.removeprefix("BigCodeBench/") for task_id in task_ids])
+        return self.client.predict(
+            split=self.SPLIT,
+            subset=self.SUBSET,
+            samples=handle_file(filename),
+            pass_k=self.PASS_K,
+            api_name="/predict",
+            selective_evaluate=selective_evaluate,
+        )
+
+    def annotate_all(self, request_states: List[RequestState]) -> List[Dict[str, Any]]:
+        task_id_to_solution: Dict[str, str] = {}
+        for request_state in request_states:
+            assert request_state.instance.id is not None
+            task_id = request_state.instance.id
+            assert request_state.result is not None
+            assert len(request_state.result.completions) == 1
+            model_output_text = request_state.result.completions[0].text
+            solution = code_extract(model_output_text)
+            task_id_to_solution[task_id] = solution
+
+        with tempfile.NamedTemporaryFile(suffix=".jsonl") as temp_file:
+            hlog(f"Temporary file for BigCodeBenchAnnotator: {temp_file.name}")
+            with open(temp_file.name, "w") as f:
+                for task_id, solution in task_id_to_solution.items():
+                    json.dump({"task_id": task_id, "solution": solution}, f)
+                    f.write("\n")
+            eval_result = self.send_request_to_gradio_evaluator(temp_file.name, list(task_id_to_solution.keys()))[0]
+        return [
+            {"bigcodebench": {"pass_at_one": eval_result["eval"][request_state.instance.id][0]["status"] == "pass"}}
+            for request_state in request_states
+        ]

helm/benchmark/annotation/bird_sql_annotator.py

@@ -0,0 +1,58 @@
+from typing import Any, List, Optional
+import os
+import re
+import sqlite3
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.benchmark.runner import get_benchmark_output_path
+from helm.common.hierarchical_logger import hwarn
+
+
+class BirdSQLAnnotator(Annotator):
+    """The Bird-SQL evaluator that computes execution accuracy."""
+
+    name = "bird_sql"
+
+    def get_database_path(self, database_name: str) -> str:
+        databases_root_path = os.path.join(
+            get_benchmark_output_path(), "scenarios", "bird_sql", "dev", "unzipped_dev_databases", "dev_databases"
+        )
+        return os.path.join(databases_root_path, database_name, f"{database_name}.sqlite")
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.instance.extra_data is not None
+        database_name = request_state.instance.extra_data["db_id"]
+        print(self.get_database_path(database_name))
+        conn = sqlite3.connect(self.get_database_path(database_name))
+
+        assert len(request_state.instance.references) == 1
+        ground_truth_sql = request_state.instance.references[0].output.text
+        ground_truth_result: List[str] = []
+        try:
+            cursor = conn.cursor()
+            cursor.execute(ground_truth_sql)
+            ground_truth_result = cursor.fetchall()
+        except (sqlite3.OperationalError, sqlite3.Warning) as e:
+            hwarn(f"Ground truth SQL failed with error: {e}")
+
+        assert request_state.result is not None
+        assert len(request_state.result.completions) == 1
+        predicted_text = request_state.result.completions[0].text
+        predicted_sql_match = re.search(r"<\s*sql\s*>(.*?)<\/?\s*sql\s*>", predicted_text, re.DOTALL | re.IGNORECASE)
+        predicted_sql = predicted_sql_match.group(1) if predicted_sql_match else ""
+        predicted_result: List[str] = []
+        query_error: Optional[str] = None
+        # TODO: Run SQL queries with a timeout
+        try:
+            cursor = conn.cursor()
+            cursor.execute(predicted_sql)
+            predicted_result = cursor.fetchall()
+        except (sqlite3.OperationalError, sqlite3.Warning) as e:
+            query_error = str(e)
+
+        return {
+            "predicted_result": predicted_result,
+            "ground_truth_result": ground_truth_result,
+            "query_error": query_error,
+        }

helm/benchmark/annotation/call_center_annotator.py

@@ -0,0 +1,258 @@
+import json
+from json.decoder import JSONDecodeError
+import textwrap
+from typing import Any
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.clients.auto_client import AutoClient
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import Request
+
+
+class CallCenterSummarizationAnnotator(Annotator):
+    """Annotator for call center summarization."""
+
+    name = "call_center_summarization"
+
+    PROMPT_TEMPLATE = """\
+    Score the summary by scoring it on a five-point scale between 1 and 5 using three criteria: Faithfulness, Relevance, and Coherence.
+
+    ### Criteria
+    Faithfulness: Can all the information expressed by the summary can be inferred from the source? (1 = not at all, 5 = very much)
+    Relevance: To what extent the summary include only important information from the source? (1 = not at all, 5 = very much)
+    Coherence: Does the summary organize the relevant information into a well-structured summary? (1 = not at all, 5 = very much)
+
+    ### Call Transcript
+    {{CALL_TRANSCRIPT}}
+
+    ### Summary
+    {{SUMMARY}}
+
+    ### Task
+    Respond with only a raw JSON object in the following format, without using Markdown formatting:
+
+    {"faithfulness": <score>, "relevance": <score>, "coherence": <score>}
+    """  # noqa: E501
+
+    CRITERIA = [
+        "faithfulness",
+        "relevance",
+        "coherence",
+    ]
+
+    def __init__(self, auto_client: AutoClient):
+        super().__init__()
+        self._auto_client = auto_client
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        call_transcript = request_state.instance.input.text
+        summary = request_state.result.completions[0].text.strip()
+        if not summary.strip():
+            hlog("Returning 0 scores due to empty response")
+            return {"faithfulness": 0, "relevance": 0, "coherence": 0}
+        annotator_prompt = (
+            textwrap.dedent(CallCenterSummarizationAnnotator.PROMPT_TEMPLATE)
+            .replace("{{CALL_TRANSCRIPT}}", call_transcript)
+            .replace("{{SUMMARY}}", summary)
+        )
+        annotator_request = Request(
+            model="openai/gpt-4o-mini-2024-07-18",
+            model_deployment="openai/gpt-4o-mini-2024-07-18",
+            prompt=annotator_prompt,
+            temperature=0.0,
+            max_tokens=256,
+        )
+        annotator_response = self._auto_client.make_request(annotator_request)
+        if not annotator_response.success:
+            raise Exception(f"Annotation request failed: {annotator_response.error}")
+        assert len(annotator_response.completions) == 1
+        annotator_response_text = annotator_response.completions[0].text
+        # OpenAI models like to surround JSON objects with ```json and ``` Markdown formatting.
+        # This strips everything outside the outermost {} brackets.
+        json_start_index = annotator_response_text.find("{")
+        json_end_index = annotator_response_text.rfind("}")
+        if json_start_index < 0 or json_end_index < 0:
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        annotator_response_json = annotator_response_text[json_start_index : json_end_index + 1]
+        try:
+            annotator_response_parsed = json.loads(annotator_response_json)
+        except JSONDecodeError:
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        for expected_key in CallCenterSummarizationAnnotator.CRITERIA:
+            if expected_key not in annotator_response_parsed:
+                raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        return annotator_response_parsed
+
+
+class CallCenterSummarizationPairwiseComparisonAnnotator(Annotator):
+    """Annotator for call center summarization with pairwise comparison."""
+
+    name = "call_center_summarization_pairwise_comparison"
+
+    PROMPT_TEMPLATE = """\
+    Given a call transcript and two different summaries of the call transcript, select your preferred summary, which can be subjective, considering the criteria below. Also provide a one-sentence reasoning for your selection.
+
+    ### Criteria
+    Faithfulness: Can all the information expressed by the summary can be inferred from the source?
+    Relevance: To what extent the summary include only important information from the source?
+    Coherence: Does the summary organize the relevant information into a well-structured summary?
+
+    ### Call Transcript
+    {{CALL_TRANSCRIPT}}
+
+    ### Summary A
+    {{SUMMARY_A}}
+
+    ### Summary B
+    {{SUMMARY_B}}
+
+    ### Task
+    Output only a JSON object with the following format:
+
+    {"reasoning": "Reasoning", "selected": "A" or "B"}
+    """  # noqa: E501
+
+    def __init__(self, auto_client: AutoClient):
+        super().__init__()
+        self._auto_client = auto_client
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        call_transcript = request_state.instance.input.text
+        summary = request_state.result.completions[0].text.strip()
+        assert len(request_state.instance.all_correct_references) == 1
+        reference_summary = request_state.instance.all_correct_references[0].output.text
+        if not summary.strip():
+            hlog("Returning 0 scores due to empty response")
+            return {"faithfulness": 0, "relevance": 0, "coherence": 0}
+        assert request_state.instance.id is not None
+        instance_id = int(request_state.instance.id[2:])
+        if instance_id % 2:
+            reference_option = "A"
+            summary_a = reference_summary
+            summary_b = summary
+        else:
+            reference_option = "B"
+            summary_a = summary
+            summary_b = reference_summary
+        annotator_prompt = (
+            textwrap.dedent(CallCenterSummarizationPairwiseComparisonAnnotator.PROMPT_TEMPLATE)
+            .replace("{{CALL_TRANSCRIPT}}", call_transcript)
+            .replace("{{SUMMARY_B}}", summary_a)
+            .replace("{{SUMMARY_A}}", summary_b)
+        )
+        annotator_request = Request(
+            model="openai/gpt-4o-2024-08-06",
+            model_deployment="openai/gpt-4o-2024-08-06",
+            prompt=annotator_prompt,
+            temperature=0.0,
+            max_tokens=256,
+        )
+        annotator_response = self._auto_client.make_request(annotator_request)
+        if not annotator_response.success:
+            raise Exception(f"Annotation request failed: {annotator_response.error}")
+        assert len(annotator_response.completions) == 1
+        annotator_response_text = annotator_response.completions[0].text
+        # OpenAI models like to surround JSON objects with ```json and ``` Markdown formatting.
+        # This strips everything outside the outermost {} brackets.
+        json_start_index = annotator_response_text.find("{")
+        json_end_index = annotator_response_text.rfind("}")
+        if json_start_index < 0 or json_end_index < 0:
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        annotator_response_json = annotator_response_text[json_start_index : json_end_index + 1]
+        try:
+            annotator_response_parsed = json.loads(annotator_response_json)
+        except JSONDecodeError:
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        for expected_key in ["reasoning", "selected"]:
+            if expected_key not in annotator_response_parsed:
+                raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        score = 0.0
+        selected = annotator_response_parsed["selected"].strip()
+        if selected != "A" and selected != "B":
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        if selected == reference_option:
+            score = 0.0
+        else:
+            score = 1.0
+        return {
+            "reasoning": annotator_response_parsed["reasoning"],
+            "selected": selected,
+            "reference_option": reference_option,
+            "score": score,
+        }
+
+
+class CallCenterSummarizationKeyPointsRecallAnnotator(Annotator):
+    """Annotator for call center summarization with key point recall."""
+
+    name = "call_center_summarization_key_points_recall"
+
+    PROMPT_TEMPLATE = """\
+    Given a call transcript, a list of key points and a summary, determine which key points are included in the summary.
+
+    ### Call Transcript
+    {{CALL_TRANSCRIPT}}
+
+    ### Key Points
+    {{KEY_POINTS}}
+
+    ### Summary
+    {{SUMMARY}}
+
+    ### Task
+    Output only a JSON array of booleans, where each boolean indicates if the corresponding key point was included in the summary.
+    """  # noqa: E501
+
+    def __init__(self, auto_client: AutoClient):
+        super().__init__()
+        self._auto_client = auto_client
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        call_transcript = request_state.instance.input.text
+        summary = request_state.result.completions[0].text.strip()
+        key_points = "\n".join(
+            [f"- {reference.output.text}" for reference in request_state.instance.all_correct_references]
+        )
+        if not summary.strip():
+            hlog("Returning 0 scores due to empty response")
+            return {"faithfulness": 0, "relevance": 0, "coherence": 0}
+        annotator_prompt = (
+            textwrap.dedent(CallCenterSummarizationKeyPointsRecallAnnotator.PROMPT_TEMPLATE)
+            .replace("{{CALL_TRANSCRIPT}}", call_transcript)
+            .replace("{{KEY_POINTS}}", key_points)
+            .replace("{{SUMMARY}}", summary)
+        )
+        annotator_request = Request(
+            model="openai/gpt-4o-2024-08-06",
+            model_deployment="openai/gpt-4o-2024-08-06",
+            prompt=annotator_prompt,
+            temperature=0.0,
+            max_tokens=256,
+        )
+        annotator_response = self._auto_client.make_request(annotator_request)
+        if not annotator_response.success:
+            raise Exception(f"Annotation request failed: {annotator_response.error}")
+        assert len(annotator_response.completions) == 1
+        annotator_response_text = annotator_response.completions[0].text
+        # OpenAI models like to surround JSON objects with ```json and ``` Markdown formatting.
+        # This strips everything outside the outermost [] brackets.
+        json_start_index = annotator_response_text.find("[")
+        json_end_index = annotator_response_text.rfind("]")
+        if json_start_index < 0 or json_end_index < 0:
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        annotator_response_json = annotator_response_text[json_start_index : json_end_index + 1]
+        try:
+            annotator_response_parsed = json.loads(annotator_response_json)
+        except JSONDecodeError:
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        if not len(annotator_response_parsed):
+            raise Exception(f"Malformed annotator response: {annotator_response_text}")
+        score = sum([1.0 if elem else 0.0 for elem in annotator_response_parsed]) / len(annotator_response_parsed)
+        return {"key_points_found": json.dumps(annotator_response_parsed), "score": score}

helm/benchmark/annotation/chw_care_plan_annotator.py

@@ -0,0 +1,82 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of a
+generated clinical note given unstructured clinical text.
+Your goal is to assess how well the generated response captures the necessary information
+and follows provided instructions in terms of accuracy, structure, and clarity.
+
+The user's request will be provided in these tags:
+<user_request>
+{QUESTION}
+</user_request>
+
+The response will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+Carefully analyze the <response>.
+For each of the following categories, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent),
+and provide a short justification for your score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Is all the information in the response factually correct?
+
+Structure (1-5)
+- Does the response contain all parts for the provided note generation structure?
+
+Clarity (1-5)
+- Is the response easy to understand for a clinician?
+
+Output Format:
+Output the evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "structure": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "structure": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class CHWCarePlanAnnotator(LLMAsJuryAnnotator):
+    """The CHWCarePlan autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="chw_care_plan",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )

helm/benchmark/annotation/czech_bank_qa_annotator.py

@@ -0,0 +1,78 @@
+import os
+import sqlite3
+import threading
+from typing import Any, Optional, Tuple
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class CzechBankQAAnnotator(Annotator):
+    """The CzechBankQA autograder.
+
+    MUST BE RUN WITH --num-threads 1 FOR SOME REASON"""
+
+    name = "czech_bank_qa"
+
+    DATABASE_SOURCE_URL = (
+        "https://huggingface.co/datasets/yifanmai/czech_bank_qa/resolve/main/czech_bank.db?download=true"
+    )
+
+    def __init__(self, file_storage_path: str):
+        super().__init__()
+
+        cache_dir = os.path.join(file_storage_path, "data")
+        ensure_directory_exists(cache_dir)
+        file_name = "czech_bank.db"
+        file_path = os.path.join(cache_dir, file_name)
+        ensure_file_downloaded(source_url=CzechBankQAAnnotator.DATABASE_SOURCE_URL, target_path=file_path)
+        database = sqlite3.connect(file_path)
+
+        # csv_files_dir = "/home/yifanmai/oss/helm-scenarios/1999-czech-bank"
+        # # table_name_to_file_name = {
+        # #     "account": "account.csv",
+        # #     "client": "client.csv",
+        # #     "disposition": "disp.csv",
+        # #     "permenant_order": "order.csv",
+        # #     "transaction": "trans.csv",
+        # #     "loan": "loan.csv",
+        # #     "credit_card": "card.csv",
+        # #     "demographic_data": "district.csv"
+        # # }
+        # for file_name in os.listdir(csv_files_dir):
+        #     file_path = os.path.join(csv_files_dir, file_name)
+        #     df = pd.read_csv(file_path)
+        #     table_name = file_name.removesuffix(".csv")
+        #     df.to_sql(table_name, database, if_exists="append", index=False)
+        #     print("Commited to SQL")
+        # # df.to_sql(table_name, conn, if_exists='append', index=False)
+
+        self.database = database
+        self.lock = threading.Lock()
+
+    def get_result(self, query: str) -> Tuple[Optional[str], Optional[str]]:
+        result: Optional[str] = None
+        error: Optional[str] = None
+        try:
+            cursor = self.database.cursor()
+            cursor.execute("PRAGMA query_only = TRUE")
+            cursor.execute(query)
+            result = str(cursor.fetchall())
+            cursor.close()
+        except (sqlite3.DatabaseError, sqlite3.Warning) as e:
+            error = str(e)
+        return (result, error)
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+
+        assert len(request_state.instance.references) == 1
+        gold_query = request_state.instance.references[0].output.text
+        query = request_state.result.completions[0].text
+        query = query.replace("```sql", "").replace("```", "")
+        result, error = self.get_result(query)
+        gold_result, gold_error = self.get_result(gold_query)
+
+        return {"query": query, "result": result, "error": error, "gold_result": gold_result, "gold_error": gold_error}

helm/benchmark/annotation/dischargeme_annotator.py

@@ -0,0 +1,96 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert responsible for evaluating a hospital document.
+The task requires generating either discharge instructions or a brief hospital course based
+on the provided discharge summary and radiology report.
+
+Your goal is to assess whether the generated text is clinically accurate, complete, and clear
+for the intended document type. The evaluation should ensure the document aligns with the
+gold response in terms of accuracy, completeness, and clarity.
+
+The target task of either generating a discharge instruction or brief hospital course along with
+the patient discharge text and radiology report will be provided in these tags:
+<patient_information>
+{QUESTION}
+</patient_information>
+
+
+The document will be provided in these tags:
+<response>
+{RESPONSE}
+</response>
+
+The gold standard target document (either discharge instructions or a brief hospital course)
+will be provided in these tags:
+<gold_response>
+{GOLD_RESPONSE}
+</gold_response>
+
+Carefully analyze the <response> based on the <patient_information> and compare
+it to the <gold_response> when necessary.
+
+For each of the following categories, rate the response on a scale of 1 to 5 (1 = very poor, 5 = excellent)
+and provide a brief justification for the score.
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Does the document provide correct medical information based on the patient's condition and source materials?
+
+Completeness (1-5)
+- Does the document include all important information needed for the specific document type?
+
+Clarity (1-5)
+- Is the document easy to understand for the right audience — patients for discharge
+  instructions or clinicians for the hospital course?
+
+Output Format:
+Generate a valid JSON object with the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Explain why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+
+class DischargeMeAnnotator(LLMAsJuryAnnotator):
+    """The DischargeMe autograder."""
+
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name="dischargeme",
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=annotator_models,
+        )