crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/annotation/omni_math_annotator.py

@@ -0,0 +1,132 @@
+from typing import Any, Dict, Optional, Union
+from importlib.resources import files
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
+from helm.clients.auto_client import AutoClient
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import Request
+
+
+# Following https://github.com/KbsdJames/Omni-MATH/blob/23be225c8e268df51990f6c5c1448f34d3b56911/GPT_eval/get_result.py
+def _parse_report(report):
+    parts = report.split("## ")
+    data = {}
+    for part in parts[1:]:
+        lines = part.strip().split("\n")
+        title = lines[0].strip()
+        content = "\n".join(lines[1:]).strip()
+        if title == "Justification":
+            data[title] = content
+        else:
+            data[title] = lines[1].strip() if len(lines) > 1 else ""
+    return data
+
+
+class OmniMATHAnnotator(Annotator):
+    """The Omni-MATH autograder."""
+
+    name = "omni_math"
+
+    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+        self._auto_client = auto_client
+        self._template_name = template_name or "gpt_evaluation_zero_shot_template"
+        template_path = files("helm.benchmark.annotation.omni_math").joinpath(f"{self._template_name}.txt")
+        with template_path.open("r") as file:
+            self._score_template = file.read()
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        prompt_template = self._score_template
+        model_output_text = request_state.result.completions[0].text
+        annotator_prompt = (
+            prompt_template.replace("{{Problem}}", request_state.instance.input.text)
+            .replace("{{Reference Answer}}", request_state.instance.references[0].output.text)
+            .replace("{{Solution}}", model_output_text)
+        )
+        if not model_output_text.strip():
+            hlog(
+                "WARNING: OmniMATHAnnotator skipped sending requests to annotator models "
+                "because the model response was empty"
+            )
+            return {
+                "prompt_text": None,
+                "empty_output_equivalence_judgement": False,
+            }
+
+        SHORT_NAME_TO_MODEL_INFO: Dict[str, AnnotatorModelInfo] = {
+            "gpt": AnnotatorModelInfo(
+                model_name="openai/gpt-4o-2024-05-13",
+                model_deployment="openai/gpt-4o-2024-05-13",
+            ),
+            "llama": AnnotatorModelInfo(
+                model_name="meta/llama-3.1-405b-instruct-turbo",
+                model_deployment="together/llama-3.1-405b-instruct-turbo",
+            ),
+            "claude": AnnotatorModelInfo(
+                model_name="anthropic/claude-3-5-sonnet-20241022",
+                model_deployment="anthropic/claude-3-5-sonnet-20241022",
+            ),
+        }
+        annotations: Dict[str, Union[Optional[str], Optional[bool]]] = {"prompt_text": annotator_prompt}
+
+        for annotator_name, annotator_model_info in SHORT_NAME_TO_MODEL_INFO.items():
+            student_final_answer: Optional[str] = None
+            equivalence_judgement: Optional[bool] = None
+            justification: Optional[str] = None
+            annotator_request = Request(
+                model=annotator_model_info.model_name,
+                model_deployment=annotator_model_info.model_deployment,
+                prompt=annotator_prompt,
+                temperature=0.0,
+                max_tokens=4096,
+            )
+            annotator_response = self._auto_client.make_request(annotator_request)
+            if not annotator_response.success:
+                hlog(
+                    "WARNING: OmniMATHAnnotator got an error response from "
+                    f"{annotator_model_info.model_name}: {annotator_response.error}"
+                )
+            else:
+                assert len(annotator_response.completions) == 1
+                annotator_response_text = annotator_response.completions[0].text
+                report_parts: Dict[str, str] = _parse_report(annotator_response_text)
+                try:
+                    student_final_answer = report_parts["Student Final Answer"]
+                except KeyError:
+                    hlog(
+                        "WARNING: OmniMATHAnnotator could not get Student Final Answer from annotation from "
+                        f"{annotator_model_info.model_name}: {annotator_response_text}"
+                    )
+
+                try:
+                    justification = report_parts["Justification"].strip().removesuffix("=== report over ===").strip()
+                except KeyError:
+                    hlog(
+                        "WARNING: OmniMATHAnnotator could not get Justification from annotation from "
+                        f"{annotator_model_info.model_name}: {annotator_response_text}"
+                    )
+
+                try:
+                    equivalence_judgement_str = report_parts["Equivalence Judgement"].strip().upper()
+                    if equivalence_judgement_str == "TRUE":
+                        equivalence_judgement = True
+                    elif equivalence_judgement_str == "FALSE":
+                        equivalence_judgement = False
+                    else:
+                        hlog(
+                            "WARNING: OmniMATHAnnotator got a non-boolean Equivalence Judgement from annotation from "
+                            f"{annotator_model_info.model_name}: {equivalence_judgement_str}"
+                        )
+                except KeyError:
+                    hlog(
+                        "WARNING: OmniMATHAnnotator could not get Equivalence Judgement from annotation from "
+                        f"{annotator_model_info.model_name}: {annotator_response_text}"
+                    )
+
+            annotations[f"{annotator_name}_student_final_answer"] = student_final_answer
+            annotations[f"{annotator_name}_justification"] = justification
+            annotations[f"{annotator_name}_equivalence_judgement"] = equivalence_judgement
+        return annotations

helm/benchmark/annotation/spider_annotator.py

@@ -0,0 +1,18 @@
+import os
+
+from helm.benchmark.annotation.bird_sql_annotator import BirdSQLAnnotator
+from helm.benchmark.runner import get_benchmark_output_path
+
+
+class SpiderAnnotator(BirdSQLAnnotator):
+    """The Spider evaluator that computes execution accuracy.
+
+    Based on the Bird-SQL annotator."""
+
+    name = "spider"
+
+    def get_database_path(self, database_name: str) -> str:
+        databases_root_path = os.path.join(
+            get_benchmark_output_path(), "scenarios", "spider", "data", "spider_data", "test_database"
+        )
+        return os.path.join(databases_root_path, database_name, f"{database_name}.sqlite")

helm/benchmark/annotation/starr_patient_instructions_annotator.py

@@ -0,0 +1,98 @@
+from typing import Dict, Optional, Set
+
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo, LLMAsJuryAnnotator
+from helm.clients.auto_client import AutoClient
+
+
+PROMPT_TEMPLATE = """You are a medical expert tasked with evaluating the quality of after-surgery care instructions.
+
+Your goal is to assess how well the care instructions capture the next steps the patient should follow after their
+procedure, and how it compares to the gold response in terms of accuracy, completeness, and clarity.
+
+The case will be provided in these tags:
+<case_details>
+{{QUESTION}}
+</case_details>
+
+The response will be provided in these tags:
+<response>
+{{RESPONSE}}
+</response>
+
+The reference response will be provided in these tags:
+<gold_response>
+{{GOLD_RESPONSE}}
+</gold_response>
+
+Carefully review the <response> and compare it to the <gold_response> when needed.
+
+Please evaluate the instructions on a scale of 1-5 (1 = poor, 5 = excellent) for each of these three key dimensions:
+
+Evaluation Criteria:
+Accuracy (1-5)
+- Are the instructions medically correct for the patient's specific case?
+
+Completeness (1-5)
+- Do the instructions cover all important post-surgical care details?
+
+Clarity (1-5)
+- Are the instructions easy for the patient to understand?
+
+Output Format:
+Output your evaluation as a single valid JSON object matching the following structure:
+{
+    "accuracy": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "completeness": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    },
+    "clarity": {
+        "score": 0,
+        "explanation": "Brief explanation of why this score was given."
+    }
+}
+
+Ensure the output is valid JSON:
+- Use **double quotes** (") for all keys and string values.
+- When quoting text or sections inside the explanations, use escaped double quotes (\") to
+  maintain valid JSON formatting.
+- Do not include any additional information in the output.
+"""
+
+ANNOTATION_CRITERIA: Dict[str, Set[str]] = {
+    "accuracy": {"score", "explanation"},
+    "completeness": {"score", "explanation"},
+    "clarity": {"score", "explanation"},
+}
+
+ANNOTATOR_MODELS: Dict[str, AnnotatorModelInfo] = {
+    "gpt": AnnotatorModelInfo(
+        model_name="openai/gpt-4o-2024-05-13",
+        model_deployment="stanfordhealthcare/gpt-4o-2024-05-13",
+    ),
+    "llama": AnnotatorModelInfo(
+        model_name="meta/llama-3.3-70b-instruct",
+        model_deployment="stanfordhealthcare/llama-3.3-70b-instruct",
+    ),
+    "claude": AnnotatorModelInfo(
+        model_name="anthropic/claude-3-7-sonnet-20250219",
+        model_deployment="stanfordhealthcare/claude-3-7-sonnet-20250219",
+    ),
+}
+
+
+class StarrPatientInstructionsAnnotator(LLMAsJuryAnnotator):
+    """The StarrPatientInstructions autograder."""
+
+    name = "starr_patient_instructions"
+
+    def __init__(self, auto_client: AutoClient, template_name: Optional[str] = None):
+        super().__init__(
+            auto_client=auto_client,
+            prompt_template=PROMPT_TEMPLATE,
+            annotation_criteria=ANNOTATION_CRITERIA,
+            annotator_models=ANNOTATOR_MODELS,
+        )

helm/benchmark/annotation/wildbench/eval_template.pairwise.v2.md

@@ -0,0 +1,75 @@
+# Instruction 
+
+You are an expert evaluator. Your task is to evaluate the quality of the responses generated by two AI models. 
+We will provide you with the user query and a pair of AI-generated responses (Response A and Response B). 
+You should first read the user query and the conversation history carefully for analyzing the task, and then evaluate the quality of the responses based on and rules provided below.
+
+# Conversation between User and AI
+
+## History
+<|begin_of_history|>
+
+{$history}
+
+<|end_of_history|> 
+
+## Current User Query
+<|begin_of_query|>
+
+{$user_query}
+
+<|end_of_query|>
+
+## Response A
+<|begin_of_response_A|>
+
+{$candidate_A}
+
+<|end_of_response_A|>
+
+## Response B
+<|begin_of_response_B|>
+
+{$candidate_B}
+
+<|end_of_response_B|>
+
+# Evaluation   
+
+## Checklist 
+
+<|begin_of_checklist|>
+
+{$checklist}
+
+<|end_of_checklist|>
+
+Please use this checklist to guide your evaluation, but do not limit your assessment to the checklist.
+
+## Rules 
+
+You should compare the above two responses based on your analysis of the user queries and the conversation history.
+You should first write down your analysis and the checklist that you used for the evaluation, and then provide your assessment according to the checklist.
+There are five choices to give your final assessment: ["A++", "A+", "A=B", "B+", "B++"], which correspond to the following meanings:
+
+- `A++`: Response A is much better than Response B.
+- `A+`: Response A is only slightly better than Response B.
+- `A=B`: Response A and B are of the same quality. Please use this choice sparingly.
+- `B+`: Response B is only slightly better than Response A.
+- `B++`: Response B is much better than Response A.
+
+
+## Output Format 
+First, please output your analysis for each model response, and then summarize your assessment to three aspects: "reason A=B", "reason A>B", and "reason B>A", and finally make your choice for the final assessment.
+
+Please provide your evaluation results in the following json format by filling in the placeholders in []:
+```
+{
+    "analysis of A": "[analysis of Response A]",
+    "analysis of B": "[analysis of Response B]",
+    "reason of A=B": "[where Response A and B perform equally well]",
+    "reason of A>B": "[where Response A is better than Response B]",
+    "reason of B>A": "[where Response B is better than Response A]",
+    "choice": "[A++ or A+ or A=B or B+ or B++]",
+}
+```

helm/benchmark/annotation/wildbench/eval_template.score.v2.md

@@ -0,0 +1,66 @@
+# Instruction 
+
+You are an expert evaluator. Your task is to evaluate the quality of the responses generated by AI models. 
+We will provide you with the user query and an AI-generated responses.
+You should first read the user query and the conversation history carefully for analyzing the task, and then evaluate the quality of the responses based on and rules provided below.
+
+# Conversation between User and AI
+
+## History
+<|begin_of_history|>
+
+{$history}
+
+<|end_of_history|> 
+
+## Current User Query
+<|begin_of_query|>
+
+{$user_query}
+
+<|end_of_query|>
+
+## AI Response
+<|begin_of_response|>
+
+{$model_output}
+
+<|end_of_response|>
+ 
+
+# Evaluation   
+
+## Checklist 
+
+<|begin_of_checklist|>
+
+{$checklist}
+
+<|end_of_checklist|>
+
+Please use this checklist to guide your evaluation, but do not limit your assessment to the checklist.
+
+## Rules 
+
+You should compare the above response based on your analysis of the user queries and the conversation history.
+You should first write down your analysis and the checklist that you used for the evaluation, and then provide your assessment according to the checklist.
+The scores are in the range of 1~10, where 1 means the response is very poor and 10 means the response is perfect.
+Here are more detailed criteria for the scores:
+
+- Score 1~2: The response is very poor and does not make sense at all.
+- Score 3~4: The response is poor and does help user solve the problem in a meaningful way.
+- Score 5~6: The response is fair but has some issues (e.g., factual errors, hallucinations, missing key information).
+- Score 7~8: The response is good enough but could be improved in some ways.
+- Score 9~10: The response is perfect and provides helpful information that can help user solve the problem.
+
+## Output Format 
+First, please output your analysis for the model response, and then summarize your assessment to two aspects: "strengths" and "weaknesses"; Finally, please write down your rating for the assessment.
+
+Please provide your evaluation results in the following json format by filling in the placeholders in []:
+```
+{
+    "strengths": "[analysis for the strengths of the response]",
+    "weaknesses": "[analysis for the weaknesses of the response]",
+    "score": "[1~10]"
+}
+```

helm/benchmark/annotation/wildbench_annotator.py

@@ -0,0 +1,119 @@
+import re
+from typing import Any, Optional, Union
+from importlib.resources import files
+from typing import Dict
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import Annotator
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
+from helm.clients.auto_client import AutoClient
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import Request
+
+
+class WildBenchAnnotator(Annotator):
+    """The WildBench autograder."""
+
+    name = "wildbench"
+
+    def __init__(self, auto_client: AutoClient):
+        self._auto_client = auto_client
+        template_path = files("helm.benchmark.annotation.wildbench").joinpath("eval_template.score.v2.md")
+        with template_path.open("r") as f:
+            self._score_template = f.read()
+        self._pattern = re.compile(
+            r'"strengths"\s*:\s*"(.*?)"\s*,\s*"weaknesses"\s*:\s*"(.*?)"\s*,\s*"score"\s*:\s*(".*?"|\d+)', re.DOTALL
+        )
+
+    def annotate(self, request_state: RequestState) -> Any:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        assert request_state.instance.extra_data
+        model_output_text = request_state.result.completions[0].text
+        if not model_output_text.strip():
+            # Following https://github.com/allenai/WildBench/blob/d6b8dcaf377d173d031980f97c16e1a82618c03d/src/eval.py
+            hlog(
+                "WARNING: WildBenchAnnotator skipped sending requests to annotator models "
+                "because the model response was empty"
+            )
+            return {
+                "prompt_text": None,
+                "empty_output_score": 1.0,
+            }
+
+        input_messages = request_state.instance.input.messages
+        assert input_messages is not None
+        history = []
+        for round in input_messages[:-1]:
+            noun = "USER: " if round["role"] == "user" else "ASSISTANT: "
+            history.append(noun + round["content"])
+        history_text = "\n\n".join(history)
+        user_query_text = input_messages[-1]["content"]
+        checklist_text = "\n".join(
+            [f"- {checklist_item}" for checklist_item in request_state.instance.extra_data["checklist"]]
+        )
+        annotator_prompt = (
+            self._score_template.replace("{$history}", history_text)
+            .replace("{$user_query}", user_query_text)
+            .replace("{$model_output}", model_output_text)
+            .replace("{$checklist}", checklist_text)
+        )
+
+        SHORT_NAME_TO_MODEL_INFO: Dict[str, AnnotatorModelInfo] = {
+            "gpt": AnnotatorModelInfo(
+                model_name="openai/gpt-4o-2024-05-13",
+                model_deployment="openai/gpt-4o-2024-05-13",
+            ),
+            "llama": AnnotatorModelInfo(
+                model_name="meta/llama-3.1-405b-instruct-turbo",
+                model_deployment="together/llama-3.1-405b-instruct-turbo",
+            ),
+            "claude": AnnotatorModelInfo(
+                model_name="anthropic/claude-3-5-sonnet-20241022",
+                model_deployment="anthropic/claude-3-5-sonnet-20241022",
+            ),
+        }
+        annotations: Dict[str, Union[Optional[str], Optional[float]]] = {"prompt_text": annotator_prompt}
+        for annotator_name, annotator_model_info in SHORT_NAME_TO_MODEL_INFO.items():
+            annotator_request = Request(
+                model=annotator_model_info.model_name,
+                model_deployment=annotator_model_info.model_deployment,
+                prompt=annotator_prompt,
+                temperature=0.0,
+                max_tokens=2000,
+            )
+            strengths: Optional[str] = None
+            weaknesses: Optional[str] = None
+            score: Optional[float] = None
+            annotator_response = self._auto_client.make_request(annotator_request)
+            if not annotator_response.success:
+                hlog(
+                    "WARNING: WildBenchAnnotator got an error response from "
+                    f"{annotator_model_info.model_name}: : {annotator_response.error}"
+                )
+            else:
+                assert len(annotator_response.completions) == 1
+                annotator_response_text = annotator_response.completions[0].text
+                annotator_response_parts = self._pattern.search(annotator_response_text)
+                if not annotator_response_parts:
+                    hlog(
+                        "WARNING: WildBenchAnnotator got a malformed annotation from "
+                        f"{annotator_model_info.model_name}: {annotator_response_text}"
+                    )
+                else:
+                    strengths = annotator_response_parts[1].strip()
+                    weaknesses = annotator_response_parts[2].strip()
+                    score_text = annotator_response_parts[3].strip().strip('"')
+                    try:
+                        score = float(score_text)
+                    except ValueError:
+                        hlog(
+                            "WARNING: WildBenchAnnotator could not parse the score from the annotation from "
+                            f"{annotator_model_info.model_name}: {annotator_response_text}"
+                        )
+
+            annotations[f"{annotator_name}_strengths"] = strengths
+            annotations[f"{annotator_name}_weaknesses"] = weaknesses
+            annotations[f"{annotator_name}_score"] = score
+
+        return annotations

helm/benchmark/annotation_executor.py

@@ -13,7 +13,7 @@ from helm.common.general import ensure_directory_exists, parallel_map, get_crede
 from helm.common.hierarchical_logger import htrack, hlog
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.annotation.annotator import AnnotatorSpec, Annotator
+from helm.benchmark.annotation.annotator import Annotator
 from helm.benchmark.annotation.annotator_factory import AnnotatorFactory
 from helm.proxy.services.service import CACHE_DIR
 
@@ -88,22 +88,33 @@ class AnnotationExecutor:
             hlog("Skipped annotation.")
             return scenario_state
 
-        if scenario_state.annotator_specs is None or len(scenario_state.annotator_specs) == 0:
+        if not scenario_state.annotator_specs:
             hlog("No annotators to run.")
             return scenario_state
 
-        # Do it!
-        def do_it(request_state: RequestState) -> RequestState:
-            assert scenario_state.annotator_specs is not None
-            return self.process(scenario_state.annotator_specs, request_state)
+        try:
+            annotators: List[Annotator] = [
+                self.factory.get_annotator(annotator_spec) for annotator_spec in scenario_state.annotator_specs
+            ]
+        except Exception as e:
+            raise AnnotationExecutorError(f"Could not initialize annotator for spec: {str(e)} ") from e
 
-        self.annotator_specs = scenario_state.annotator_specs
+        if all(getattr(annotator, "use_global_metric", False) for annotator in annotators):
+            # Do it!
+            request_states = self.process_all(
+                annotators, scenario_state.request_states  # processing all request together
+            )
 
-        request_states = parallel_map(
-            do_it,
-            scenario_state.request_states,
-            parallelism=self.execution_spec.parallelism,
-        )
+        else:
+            # Do it!
+            def do_it(request_state: RequestState) -> RequestState:
+                return self.process(annotators, request_state)
+
+            request_states = parallel_map(
+                do_it,
+                scenario_state.request_states,
+                parallelism=self.execution_spec.parallelism,
+            )
 
         hlog(f"Annotated {len(request_states)} requests")
         return ScenarioState(
@@ -112,13 +123,22 @@ class AnnotationExecutor:
             annotator_specs=scenario_state.annotator_specs,
         )
 
-    def process(self, annotator_specs: List[AnnotatorSpec], state: RequestState) -> RequestState:
+    def process(self, annotators: List[Annotator], state: RequestState) -> RequestState:
         annotations: Dict[str, Any] = {}
         try:
-            for annotator_spec in annotator_specs:
-                annotator: Annotator = self.factory.get_annotator(annotator_spec)
+            for annotator in annotators:
                 new_annotations = annotator.annotate(state)
                 annotations[annotator.name] = new_annotations
         except Exception as e:
             raise AnnotationExecutorError(f"{str(e)} Request: {state.request}") from e
         return replace(state, annotations=annotations)
+
+    def process_all(self, annotators: List[Annotator], states: List[RequestState]) -> List[RequestState]:
+        annotations: Dict[str, Any] = {}
+        try:
+            for annotator in annotators:
+                new_annotations = annotator.annotate_all(states)
+                annotations[annotator.name] = new_annotations
+        except Exception as e:
+            raise AnnotationExecutorError(f"{str(e)} Request: {[state.request for state in states]}") from e
+        return [replace(state, annotations=new_annotations[idx]) for idx, state in enumerate(states)]

helm/benchmark/augmentations/cleva_perturbation.py

@@ -9,8 +9,9 @@ from typing import Dict, List, Tuple, Set, Optional
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
-from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation, TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import Perturbation, TextPerturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 ############################################################
@@ -69,7 +70,7 @@ class ChineseTyposPerturbation(TextPerturbation):
         self.word_level_perturb: bool = word_level_perturb  # Whether we perturb text on the character or word level
 
         # Ensure all necessary data are downloaded
-        output_dir = os.path.join("benchmark_output", "perturbations", self.name)
+        output_dir = os.path.join(get_benchmark_output_path(), "perturbations", self.name)
         ensure_directory_exists(output_dir)
         for filename in self.FILE_NAMES:
             target_path = os.path.join(output_dir, filename)
@@ -303,7 +304,7 @@ class ChineseSynonymPerturbation(TextPerturbation):
         self.prob: float = prob
         self.trial_num: int = trial_num  # Number of trial to get a 100% perturbed text
 
-        target_dir = os.path.join("benchmark_output", "perturbations", self.name, "synonyms.json")
+        target_dir = os.path.join(get_benchmark_output_path(), "perturbations", self.name, "synonyms.json")
         ensure_directory_exists(os.path.dirname(target_dir))
         ensure_file_downloaded(source_url=self.SOURCE_URL, target_path=target_dir)
         with open(os.path.join(target_dir)) as f:
@@ -433,7 +434,7 @@ class ChineseGenderPerturbation(TextPerturbation):
         if self.mode == self.GENDER_TERM:
             self.term_dict: Dict[Tuple[str, str], Dict[str, str]] = defaultdict(dict)
 
-            target_path = os.path.join("benchmark_output", "perturbations", self.name, "gender_term.txt")
+            target_path = os.path.join(get_benchmark_output_path(), "perturbations", self.name, "gender_term.txt")
             ensure_directory_exists(os.path.dirname(target_path))
             ensure_file_downloaded(source_url=self.SOURCE_URL, target_path=target_path)
             with open(target_path) as fin:
@@ -492,7 +493,7 @@ class ChinesePersonNamePerturbation(Perturbation):
 
     """ Resources """
     SOURCE_URL: str = "http://39.108.215.175/assets/chinese_name_gender.json"
-    OUTPUT_PATH = os.path.join("benchmark_output", "perturbations", name)
+    OUTPUT_PATH = os.path.join(get_benchmark_output_path(), "perturbations", name)
 
     """ Gender categories """
     GENDER_CATEGORY = "gender"
@@ -554,7 +555,7 @@ class ChinesePersonNamePerturbation(Perturbation):
 
         self.preserve_gender: bool = preserve_gender
 
-        target_path = os.path.join("benchmark_output", "perturbations", self.name, "chinese_name_gender.json")
+        target_path = os.path.join(get_benchmark_output_path(), "perturbations", self.name, "chinese_name_gender.json")
         ensure_directory_exists(os.path.dirname(target_path))
         ensure_file_downloaded(source_url=self.SOURCE_URL, target_path=target_path)
         with open(os.path.join(target_path), "r", encoding="utf-8") as f:
@@ -735,7 +736,7 @@ class MandarinToCantonesePerturbation(TextPerturbation):
             handle_module_not_found_error(e, ["cleva"])
         self.s2t_converter = opencc.OpenCC("s2t.json")
 
-        target_path = os.path.join("benchmark_output", "perturbations", self.name, "conversion.json")
+        target_path = os.path.join(get_benchmark_output_path(), "perturbations", self.name, "conversion.json")
         ensure_directory_exists(os.path.dirname(target_path))
         ensure_file_downloaded(source_url=self.SOURCE_URL, target_path=target_path)
         with open(target_path) as fin:

helm/benchmark/augmentations/contraction_expansion_perturbation.py

@@ -5,8 +5,8 @@ import re
 from random import Random
 
 from helm.common.general import match_case
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 CONTRACTION_MAP: Dict[str, str] = {