crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/adaptation/adapter_spec.py

@@ -63,6 +63,11 @@ class AdapterSpec:
     reference_prefix: str = "A. "
     """The string that is included before each reference (for multiple-choice questions)."""
 
+    # Set hash=False to make `AdapterSpec` hashable
+    reference_prefix_characters: Optional[List[str]] = field(default=None, hash=False)
+    """The characters that are used to identify choices for multiple-choice questions e.g. ["A", "B", "C", "D"].
+    If unset, defaults to the sequence of ascending characters starting from the first character of reference_prefix."""
+
     reference_suffix: str = "\n"
     """The string that is included after each reference (for multiple-choice questions)."""
 
@@ -139,3 +144,8 @@ class AdapterSpec:
     # Set hash=False to make `AdapterSpec` hashable
     eval_splits: Optional[List[str]] = field(default=None, hash=False)
     """The splits from which evaluation instances will be drawn."""
+
+    output_mapping_pattern: Optional[str] = None
+    """Pattern to apply to the output before applying the output mapping for the joint multiple choice adapter.
+    If the pattern has no group, the output mapping will be applied to the first match.
+    If the pattern has a group, the output mapping will be applied to the group of the first match."""

helm/benchmark/adaptation/adapters/multimodal/multiple_choice_joint_multimodal_adapter.py

@@ -18,12 +18,20 @@ class MultipleChoiceJointMultimodalAdapter(InContextLearningMultimodalAdapter, A
     learning for multimodal models.
     """
 
-    @staticmethod
-    def get_reference_prefix(prefix: str, i: int) -> str:
+    def get_prefix_char(self, prefix: str) -> str:
+        return [char for char in prefix if char.isalnum()][0]
+
+    def get_reference_prefix(self, prefix: str, i: int) -> str:
         """
         Example: prefix = "\nA. ", i = 2, return "\nC. "
         """
-        return prefix.replace("A", chr(ord("A") + i))
+        old_prefix_char = self.get_prefix_char(prefix)
+        new_prefix_char = (
+            self.adapter_spec.reference_prefix_characters[i]
+            if self.adapter_spec.reference_prefix_characters
+            else chr(ord(old_prefix_char) + i)
+        )
+        return prefix.replace(old_prefix_char, new_prefix_char)
 
     def generate_requests(
         self, eval_instance: Instance, train_trial_index: int, training_instances: List[Instance]

helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py

@@ -38,22 +38,25 @@ class MultipleChoiceJointAdapter(InContextLearningAdapter):
         <input_prefix><input><reference_prefixes[index]><reference><output_prefix><output>
     """
 
-    @staticmethod
-    def get_prefix_char(prefix: str) -> str:
+    def get_prefix_char(self, prefix: str) -> str:
         return [char for char in prefix if char.isalnum()][0]
 
-    @staticmethod
-    def get_reference_prefix(prefix: str, i: int) -> str:
+    def get_reference_prefix(self, prefix: str, i: int) -> str:
         """
         Example: prefix = "\nA. ", i = 2, return "\nC. "
         """
-        prefix_char = MultipleChoiceJointAdapter.get_prefix_char(prefix)
-        return prefix.replace(prefix_char, chr(ord(prefix_char) + i))
+        old_prefix_char = self.get_prefix_char(prefix)
+        new_prefix_char = (
+            self.adapter_spec.reference_prefix_characters[i]
+            if self.adapter_spec.reference_prefix_characters
+            else chr(ord(old_prefix_char) + i)
+        )
+        return prefix.replace(old_prefix_char, new_prefix_char)
 
     def generate_requests(
         self, eval_instance: Instance, train_trial_index: int, training_instances: List[Instance]
     ) -> List[RequestState]:
-        prefix_char = MultipleChoiceJointAdapter.get_prefix_char(self.adapter_spec.reference_prefix)
+        prefix_char = self.get_prefix_char(self.adapter_spec.reference_prefix)
         prompt = self.construct_prompt(training_instances, eval_instance, include_output=False, reference_index=None)
         output_mapping: Dict[str, str] = dict(
             (self.get_reference_prefix(prefix_char, reference_index), reference.output.text)
@@ -91,7 +94,7 @@ class MultipleChoiceJointAdapter(InContextLearningAdapter):
         # Include the references
         delimiter = ", "
         no_correct_references = "n/a"
-        prefix_char = MultipleChoiceJointAdapter.get_prefix_char(self.adapter_spec.reference_prefix)
+        prefix_char = self.get_prefix_char(self.adapter_spec.reference_prefix)
         output = no_correct_references
         for reference_index, reference in enumerate(instance.references):
             prefix = self.get_reference_prefix(self.adapter_spec.reference_prefix, reference_index)

helm/benchmark/annotation/aci_bench_annotator.py

@@ -10,15 +10,15 @@ Your goal is to assess how well the note captures the clinical information from
 compare it to the reference note (gold standard) in terms of accruacy, completeness and clarity.
 The conversation will be provided in these tags:
 <conversation>
-{{QUESTION}}
+{QUESTION}
 </conversation>
 The generated note will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 The reference note will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 Carefully review the <response> based on the <conversation> and compare it to the <gold_response> when needed.
 
@@ -65,31 +65,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class ACIBenchAnnotator(LLMAsJuryAnnotator):
     """The ACIBench autograder."""
 
-    name = "aci_bench"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="aci_bench",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/air_bench_annotator.py

@@ -37,7 +37,7 @@ class AIRBench2024Annotator(Annotator):
         )
         self._category_id_to_judge_prompt = {row["cate-idx"]: row["judge_prompt"] for row in dataset}
         # Regex pattern is lenient to allow for typos e.g. extra whitespace
-        self._pattern = re.compile("##\s*short_reasoning\s*:(.*)##\s*the_score\s*:(.*)", re.DOTALL)
+        self._pattern = re.compile(r"##\s*short_reasoning\s*:(.*)##\s*the_score\s*:(.*)", re.DOTALL)
         self._model = model or self._DEFAULT_MODEL
         self._model_deployment = model_deployment or self._DEFAULT_MODEL_DEPLOYMENT
 

helm/benchmark/annotation/alrage_annotator.py

@@ -0,0 +1,90 @@
+from typing import Any
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.clients.auto_client import AutoClient
+from helm.common.hierarchical_logger import hwarn
+from helm.common.request import Request
+
+
+class ALRAGEAnnotator(Annotator):
+    """The ALRAGEAnnotator autograder."""
+
+    name = "alrage"
+
+    _ANNOTATOR_MODEL = "openai/gpt-4o-2024-11-20"
+
+    def __init__(self, auto_client: AutoClient):
+        self._auto_client = auto_client
+
+    def _apply_annotator_template(self, question: str, answer: str, gold: str) -> Any:
+        return [
+            {
+                "role": "system",
+                "content": """أنت مقيّم محايد خبير باللغة العربية. يجب عليك:
+        1. تقييم دقة الإجابة مقارنة بالإجابة الصحيحة
+        2. التحقق من أن الإجابة مدعومة بالسياق المقدم
+        3. تقييم جودة وشمولية الإجابة
+
+        مهم جداً: يجب أن يكون ردك رقماً فقط من 0 إلى 10. لا تضف أي نص أو تفسير.""",
+            },
+            {
+                "role": "user",
+                "content": f"""السؤال: {question}
+
+        الإجابة المقدمة: {answer}
+
+        الإجابة الصحيحة: {gold}
+
+        أعط تقييماً من 0 إلى 10:
+        0-2: إجابة خاطئة تماماً
+        3-4: إجابة جزئية مع أخطاء
+        5-6: إجابة متوسطة
+        7-8: إجابة جيدة
+        9-10: إجابة ممتازة
+
+        اكتب رقماً فقط من 0 إلى 10 بدون أي نص إضافي:""",
+            },
+        ]
+
+    def _parse_annotator_response(self, response: str) -> float:
+        """Process the judge's response to extract the score"""
+        try:
+            # Extract the first number from the response content
+            score = float(next(num for num in response.split() if num.replace(".", "", 1).isdigit()))
+            return min(max(score / 10.0, 0.0), 1.0)
+
+        except Exception as e:
+            hwarn(f"Error while processing judge response: {e}")
+            return 0.0
+
+    def annotate(self, request_state: RequestState) -> Any:
+        question = request_state.instance.input.text
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        answer = request_state.result.completions[0].text
+        assert len(request_state.instance.all_correct_references) == 1
+        gold = request_state.instance.all_correct_references[0].output.text
+        messages = self._apply_annotator_template(question, answer, gold)
+        judge_request = Request(
+            model=self._ANNOTATOR_MODEL,
+            model_deployment=self._ANNOTATOR_MODEL,
+            messages=messages,
+            temperature=0.0,
+            max_tokens=2000,
+        )
+        judge_response = self._auto_client.make_request(judge_request)
+        if not judge_response.success:
+            raise Exception(
+                "ALRAGEAnnotator got an error response from " f"{self._ANNOTATOR_MODEL}: {judge_response.error}"
+            )
+        assert len(judge_response.completions) == 1
+        prompt = messages[-1]["content"]
+        response = judge_response.completions[0].text
+        score = self._parse_annotator_response(response)
+
+        return {
+            "prompt": prompt,
+            "response": response,
+            "score": score,
+        }

helm/benchmark/annotation/chw_care_plan_annotator.py

@@ -11,12 +11,12 @@ and follows provided instructions in terms of accuracy, structure, and clarity.
 
 The user's request will be provided in these tags:
 <user_request>
-{{QUESTION}}
+{QUESTION}
 </user_request>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 Carefully analyze the <response>.
@@ -63,31 +63,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class CHWCarePlanAnnotator(LLMAsJuryAnnotator):
     """The CHWCarePlan autograder."""
 
-    name = "chw_care_plan"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="chw_care_plan",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/dischargeme_annotator.py

@@ -15,19 +15,19 @@ gold response in terms of accuracy, completeness, and clarity.
 The target task of either generating a discharge instruction or brief hospital course along with
 the patient discharge text and radiology report will be provided in these tags:
 <patient_information>
-{{QUESTION}}
+{QUESTION}
 </patient_information>
 
 
 The document will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The gold standard target document (either discharge instructions or a brief hospital course)
 will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully analyze the <response> based on the <patient_information> and compare
@@ -77,31 +77,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class DischargeMeAnnotator(LLMAsJuryAnnotator):
     """The DischargeMe autograder."""
 
-    name = "dischargeme"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="dischargeme",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/live_qa_annotator.py

@@ -50,7 +50,7 @@ class LiveQAAnnotator(Annotator):
         cache_dir = os.path.join(file_storage_path, "data")
         ensure_directory_exists(cache_dir)
         # Regex pattern is lenient to allow for typos e.g. extra whitespace
-        self._pattern = re.compile("##\s*short_reasoning\s*:(.*)##\s*the_score\s*:(.*)", re.DOTALL)
+        self._pattern = re.compile(r"##\s*short_reasoning\s*:(.*)##\s*the_score\s*:(.*)", re.DOTALL)
 
     def annotate(self, request_state: RequestState) -> Any:
         assert request_state.result

helm/benchmark/annotation/med_dialog_annotator.py

@@ -11,17 +11,17 @@ and how it compares to the gold response in terms of accuracy, completeness, and
 
 The patient-doctor conversation will be provided in these tags:
 <conversation>
-{{QUESTION}}
+{QUESTION}
 </conversation>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The reference response will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully review the <response> and compare it to the <gold_response> when needed.
@@ -69,31 +69,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MedDialogAnnotator(LLMAsJuryAnnotator):
     """The MedDialog autograder."""
 
-    name = "med_dialog"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="med_dialog",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/medalign_annotator.py

@@ -12,17 +12,17 @@ and aligns with the gold response in terms of accuracy, completeness, and clarit
 
 The instruction and EHR pair will be provided in these tags:
 <user_request>
-{{QUESTION}}
+{QUESTION}
 </user_request>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The gold response (reference answer) will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully review the <response> based on the <user_request> and compare it to
@@ -70,31 +70,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MedalignAnnotator(LLMAsJuryAnnotator):
     """The Medalign autograder."""
 
-    name = "medalign"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="medalign",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/medi_qa_annotator.py

@@ -11,17 +11,17 @@ and how it compares to the gold response in terms of accuracy, completeness, and
 
 The question will be provided in these tags:
 <question>
-{{QUESTION}}
+{QUESTION}
 </question>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The reference answer will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully analyze the <response> compared to the <gold_response> and the original <question>.
@@ -68,31 +68,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MediQAAnnotator(LLMAsJuryAnnotator):
     """The MediQA autograder."""
 
-    name = "medi_qa"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="medi_qa",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )

helm/benchmark/annotation/medication_qa_annotator.py

@@ -11,17 +11,17 @@ and how it compares to the gold response in terms of accuracy, completeness, and
 
 The question provided in these tags:
 <medication_question>
-{{QUESTION}}
+{QUESTION}
 </medication_question>
 
 The response will be provided in these tags:
 <response>
-{{RESPONSE}}
+{RESPONSE}
 </response>
 
 The reference response will be provided in these tags:
 <gold_response>
-{{GOLD_RESPONSE}}
+{GOLD_RESPONSE}
 </gold_response>
 
 Carefully review the <response> and compare it to the <gold_response> when needed.
@@ -67,31 +67,20 @@ ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
     "clarity": {"score", "explanation"},
 }
 
-ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
-    "gpt": AnnotatorModelInfo(
-        model_name="openai/gpt-4o-2024-05-13",
-        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
-    ),
-    "llama": AnnotatorModelInfo(
-        model_name="meta/llama-3.3-70b-instruct",
-        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
-    ),
-    "claude": AnnotatorModelInfo(
-        model_name="anthropic/claude-3-7-sonnet-20250219",
-        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
-    ),
-}
-
 
 class MedicationQAAnnotator(LLMAsJuryAnnotator):
     """The MedicationQA autograder."""
 
-    name = "medication_qa"
-
-    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+    def __init__(
+        self,
+        auto_client: AutoClient,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        template_name: Optional[str] = None,
+    ):
         super().__init__(
+            name="medication_qa",
             auto_client=auto_client,
             prompt_template=PROMPT_TEMPLATE,
             annotation_criteria=ANNOTATION_CRITERIA,
-            annotator_models=ANNOTATOR_MODELS,
+            annotator_models=annotator_models,
         )