crfm-helm 0.5.7__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/annotation/mimic_bhc_annotator.py

@@ -12,17 +12,17 @@ compares to the gold response in terms of accuracy, completeness, and clarity.
 
 The user's request will be provided in these tags:
 <user_request>
-{{QUESTION}}
+{QUESTION}
 </user_request>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 A potential correct response will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully analyze the <response>. For each of the following categories,
@@ -70,31 +70,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MIMICBHCAnnotator(LLMAsJuryAnnotator):
     """The MIMICBHC autograder."""
 
-    name = "mimic_bhc"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="mimic_bhc",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/mimic_rrs_annotator.py

@@ -11,17 +11,17 @@ how it compares to the gold response in terms of accuracy, completeness, and cla
 
 The user's request will be provided in these tags:
 <user_request>
-{{QUESTION}}
+{QUESTION}
 </user_request>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 Some potential correct responses will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully analyze the <response>.
@@ -70,31 +70,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MIMICRRSAnnotator(LLMAsJuryAnnotator):
     """The MIMICRRS autograder."""
 
-    name = "mimic_rrs"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="mimic_rrs",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/model_as_judge.py

@@ -1,6 +1,7 @@
 import json
 import re
 from dataclasses import dataclass
+from string import Template
 from typing import Dict, Optional, TypedDict, Union, Callable, Any, Set
 
 from helm.benchmark.adaptation.request_state import RequestState
@@ -113,6 +114,7 @@ class LLMAsJuryAnnotator(Annotator):
 
     def __init__(
         self,
+        name: str,
         auto_client: AutoClient,
         prompt_template: str,
         annotation_criteria: Dict[str, Set[str]],
@@ -128,6 +130,7 @@ class LLMAsJuryAnnotator(Annotator):
         :param annotator_models: Dictionary of models to use for annotation
         :param preprocessor: Optional function to preprocess model responses
         """
+        self.name = name
         self._auto_client = auto_client
         self._prompt_template = prompt_template
         self._annotation_criteria = annotation_criteria
@@ -147,32 +150,34 @@ class LLMAsJuryAnnotator(Annotator):
     def _interpolate_prompt(
         self, request_state: RequestState, custom_replacements: Optional[Dict[str, str]] = None
     ) -> str:
-        """
-        Interpolate prompt template with request state information.
-
-        :param request_state: The current request state
-        :param custom_replacements: Optional dictionary of additional replacements
-        :return: Interpolated prompt
-        """
-        base_replacements = {
-            "{{QUESTION}}": request_state.instance.input.text,
-            "{{RESPONSE}}": (
+        """Interpolate prompt templates safely, supporting {{QUESTION}}-style files."""
+        # Build required/optional fields
+        replacements: Dict[str, str] = {
+            "QUESTION": request_state.instance.input.text,
+            "RESPONSE": (
                 request_state.result.completions[0].text
                 if request_state.result and request_state.result.completions
                 else ""
             ),
-            "{{GOLD_RESPONSE}}": request_state.instance.references[0].output.text,
+            # GOLD is optional; keep empty if not present
+            "GOLD_RESPONSE": (
+                request_state.instance.references[0].output.text
+                if getattr(request_state.instance, "references", None)
+                else ""
+            ),
         }
-
-        # Allow custom replacements to override base replacements
         if custom_replacements:
-            base_replacements.update(custom_replacements)
+            replacements.update(custom_replacements)
 
-        prompt = self._prompt_template
-        for key, value in base_replacements.items():
-            prompt = prompt.replace(key, str(value))
+        tmpl_text = self._prompt_template
+
+        tmpl_text = (
+            tmpl_text.replace("{QUESTION}", "$QUESTION")
+            .replace("{RESPONSE}", "$RESPONSE")
+            .replace("{GOLD_RESPONSE}", "$GOLD_RESPONSE")
+        )
 
-        return prompt
+        return Template(tmpl_text).substitute(replacements)
 
     def _validate_annotation(self, annotator_criteria: Dict[str, Any], annotator_name: str) -> bool:
         """

helm/benchmark/annotation/mtsamples_procedures_annotator.py

@@ -11,17 +11,17 @@ and compares to the gold response in terms of accuracy, completeness, and clarit
 
 The user's request will be provided in these tags:
 <user_request>
-{{QUESTION}}
+{QUESTION}
 </user_request>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 Some potential correct responses will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully analyze the <response>.
@@ -68,31 +68,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MTSamplesProceduresAnnotator(LLMAsJuryAnnotator):
     """The MTSamplesProcedures autograder."""
 
-    name = "mtsamples_procedures"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="mtsamples_procedures",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/mtsamples_replicate_annotator.py

@@ -13,17 +13,17 @@ and aligns with the gold standard response in accuracy, completeness, and clarit
 
 The patient's information will be provided in these tags:
 <patient_information>
-{{QUESTION}}
+{QUESTION}
 </patient_information>
 
 The proposed treatment plan will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The gold standard treatment plan will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully review the <response> based on the <patient_information> and compare it to the <gold_response> when needed.
@@ -71,31 +71,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MTSamplesReplicateAnnotator(LLMAsJuryAnnotator):
     """The MTSamplesReplicate autograder."""
 
-    name = "mtsamples_replicate"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="mtsamples_replicate",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/starr_patient_instructions_annotator.py

@@ -11,17 +11,17 @@ procedure, and how it compares to the gold response in terms of accuracy, comple
 
 The case will be provided in these tags:
 <case_details>
-{{QUESTION}}
+{QUESTION}
 </case_details>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The reference response will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully review the <response> and compare it to the <gold_response> when needed.
@@ -68,31 +68,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class StarrPatientInstructionsAnnotator(LLMAsJuryAnnotator):
     """The StarrPatientInstructions autograder."""
 
-    name = "starr_patient_instructions"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="starr_patient_instructions",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )