eval-framework 0.2.0__py3-none-any.whl

eval_framework/tasks/benchmarks/gpqa.py

@@ -0,0 +1,177 @@
+import hashlib
+import logging
+import random
+import re
+from typing import Any
+
+from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
+from eval_framework.tasks.utils import get_n_letters
+
+logger = logging.getLogger(__name__)
+
+
+class GPQA(BaseTask[str]):
+    """GPQA dataset: https://huggingface.co/datasets/Idavidrein/gpqa"""
+
+    NAME = "GPQA"
+    DATASET_PATH = "Idavidrein/gpqa"
+    SAMPLE_SPLIT = "train"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = ["gpqa_extended"]  # ["gpqa_diamond", "gpqa_extended", "gpqa_main", "gpqa_experts"]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question"] + get_n_letters(4)
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+        self.stop_sequences = ["Question:"]
+        self.keys = get_n_letters(4)
+        self.num_to_letter = {str(i): letter for i, letter in enumerate(self.keys, start=1)}
+        self.rnd_choice_shuffle = random.Random(RANDOM_SEED)
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        name = subject if subject != NO_SUBJECT else None
+
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name=name)
+        self.dataset = {}
+
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            data_list = list(data)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            if split in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                # exclude in the GPQA dataset one of the sample that has an too long prompt (DNA sequence)
+                data_list_filtered = [
+                    item
+                    for item in data_list
+                    if item["Question"]
+                    != "Hello, you are embarking on a new project. You need to produce the HP1alpha protein in E. coli. Which of these plasmids will you choose?"  # noqa: E501
+                ]
+                if len(data_list) - len(data_list_filtered) > 0:
+                    logger.info(f"Excluded {len(data_list) - len(data_list_filtered)} samples from {split} split.")
+                assert len(data_list) - len(data_list_filtered) < 2, "we expect to remove max one item"
+
+                self.dataset[split] = data_list_filtered
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        system_prompt_text = (
+            "Here are some example questions from experts. "
+            "An explanation is given before the final answer. "
+            "Answer the final question yourself, giving your reasoning beforehand."
+        )
+        return system_prompt_text
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        choices, _ = self._get_possible_completions_marked(item)
+        prompt = f"Question: {item['Question'].strip()}\n"
+        prompt += "\n".join(choices) + "\n"
+        return prompt
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        choices, correct_answer_position = self._get_possible_completions_marked(item)
+        answer_key = choices[correct_answer_position][:3]
+        return f" {answer_key}"
+
+    def _get_possible_completions_marked(self, item: dict[str, Any]) -> tuple[list[str], int]:
+        choices = [self._preprocess(item[f"Incorrect Answer {x}"]) for x in range(1, 4)]
+        correct_answer = self._preprocess(item["Correct Answer"])
+        # we want to be random, but always the same for the same input
+        # so we hash the string, which always give you the same seed
+        hash_object = hashlib.sha256(f"{choices} {correct_answer}".encode())
+        self.rnd_choice_shuffle.seed(int(hash_object.hexdigest(), 16))
+        self.rnd_choice_shuffle.shuffle(choices)
+        correct_answer_position = self.rnd_choice_shuffle.randint(0, 3)
+        choices.insert(correct_answer_position, correct_answer)
+        choices = [f"({self.keys[i]}) {choice}" for i, choice in enumerate(choices)]
+        return choices, correct_answer_position
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" ({x})" for x in self.keys]
+
+    @staticmethod
+    def _preprocess(text: str | None) -> str:
+        if text is None:
+            return " "
+        text = text.strip()
+        text = text.replace(" [title]", ". ")
+        text = re.sub("\\[.*?\\]", "", text)
+        text = text.replace("  ", " ")
+        return text
+
+
+class GPQA_COT(GPQA):
+    NAME = "GPQA_COT"
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [AccuracyCompletion]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Therefore", "the", "answer", "is", "ANSWER_LETTER"] + get_n_letters(
+        4
+    )
+    ANS_RE = re.compile(r"Therefore, the answer is \(([ABCDEFGHIJ])\)")
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot == 0, "Fewshot is not supported for GPQA_COT"
+        super().__init__(num_fewshot)
+        self.stop_sequences: list[str] = ["Question:"]
+        self.keys = get_n_letters(4)
+        self.num_to_letter = {str(i): letter for i, letter in enumerate(self.keys, start=1)}
+        self.rnd_choice_shuffle = random.Random(RANDOM_SEED)
+
+    def _extract_answer(self, completion: str) -> str:
+        match = self.ANS_RE.search(completion)
+        if match:
+            match_str = match.group(1)
+            return match_str
+        else:
+            return "[invalid]"
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        for stop_sequence in self.stop_sequences:
+            if stop_sequence in completion_text:
+                completion_text = completion_text.split(stop_sequence)[0]
+        return self._extract_answer(completion_text)
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return ""
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        # using the reasoning prompt from "Figure 44 of Tülu 3 paper: https://arxiv.org/pdf/2411.15124"
+        choices, _ = self._get_possible_completions_marked(item)
+        instruction_text = (
+            "Answer the following multiple-choice question by giving the correct answer letter in parentheses. "
+            "Provide CONCISE reasoning for the answer, and make sure to finish the response with "
+            '"Therefore, the answer is (ANSWER_LETTER)" where (ANSWER_LETTER) is one of (A), (B), (C), (D), (E), etc.'
+        )
+        instruction_text += f"\n\nQuestion: {item['Question'].strip()}\n"
+        instruction_text += "\n".join(choices)
+        instruction_text += (
+            "\n\nAnswer the above question and REMEMBER to finish your response with the exact phrase "
+            '"Therefore, the answer is (ANSWER_LETTER)" where (ANSWER_LETTER) is one of (A), (B), (C), (D), (E), etc.'
+        )
+        return instruction_text
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return ""
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        choices, correct_answer_position = self._get_possible_completions_marked(item)
+        # index 1 selects the letter
+        answer_key = choices[correct_answer_position][1]
+        return answer_key

eval_framework/tasks/benchmarks/gsm8k.py

@@ -0,0 +1,148 @@
+import re
+from typing import Any
+
+from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
+from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+
+ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)")
+
+# Predefined fewshot examples
+FEWSHOT_ITEMS = [
+    {
+        "question": (
+            "There are 15 trees in the grove. Grove workers will plant trees in the grove today. "
+            "After they are done, there will be 21 trees. "
+            "How many trees did the grove workers plant today?"
+        ),
+        "answer": (
+            "There are 15 trees originally. Then there were 21 trees after some more were planted. "
+            "So there must have been 21 - 15 = 6.\n#### 6"
+        ),
+    },
+    {
+        "question": (
+            "If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?"
+        ),
+        "answer": "There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5.\n#### 5",
+    },
+    {
+        "question": (
+            "Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?"
+        ),
+        "answer": (
+            "Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. "
+            "After eating 35, they had 74 - 35 = 39.\n#### 39"
+        ),
+    },
+    {
+        "question": (
+            "Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. "
+            "How many lollipops did Jason give to Denny?"
+        ),
+        "answer": (
+            "Jason started with 20 lollipops. Then he had 12 after giving some to Denny. "
+            "So he gave Denny 20 - 12 = 8.\n#### 8"
+        ),
+    },
+    {
+        "question": (
+            "Shawn has five toys. For Christmas, he got two toys each from his mom and dad. "
+            "How many toys does he have now?"
+        ),
+        "answer": (
+            "Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. "
+            "5 + 4 = 9.\n#### 9"
+        ),
+    },
+    {
+        "question": (
+            "There were nine computers in the server room. Five more computers were installed each day, "
+            "from monday to thursday. "
+            "How many computers are now in the server room?"
+        ),
+        "answer": (
+            "There were originally 9 computers. For each of 4 days, 5 more computers were "
+            "added. So 5 * 4 = 20 computers were added. 9 + 20 is 29.\n#### 29"
+        ),
+    },
+    {
+        "question": (
+            "Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. "
+            "How many golf balls did he have at the end of wednesday?"
+        ),
+        "answer": (
+            "Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. "
+            "After losing 2 more, he had 35 - 2 = 33 golf balls.\n#### 33"
+        ),
+    },
+    {
+        "question": "Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
+        "answer": (
+            "Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. "
+            "So she has 23 - 15 dollars left. 23 - 15 is 8.\n#### 8"
+        ),
+    },
+]
+
+
+class GSM8K(BaseTask[str]):
+    """GSM8K dataset: https://huggingface.co/datasets/openai/gsm8k"""
+
+    NAME = "GSM8K"
+    DATASET_PATH = "gsm8k"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [AccuracyCompletion]
+    SUBJECTS = ["main"]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        # until: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/gsm8k/gsm8k.yaml
+        self.stop_sequences: list[str] = ["Question:"]
+        self.max_tokens = 1600
+
+    def _extract_answer(self, completion: str) -> str:
+        match = ANS_RE.search(completion)
+        if match:
+            match_str = match.group(1).strip()
+            match_str = match_str.replace(",", "")
+            return match_str
+        else:
+            return "[invalid]"
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        for stop_sequence in self.stop_sequences:
+            if stop_sequence in completion_text:
+                completion_text = completion_text.split(stop_sequence)[0]
+        return self._extract_answer(completion_text)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"Question: {item['question']}\nAnswer:"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return f" {item['answer']}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return self._extract_answer(item["answer"])
+
+
+class GSM8KLlamaVersion(GSM8K):
+    NAME = "GSM8K Llama Version"
+    FEWSHOT_SPLIT = ""  # Changed to empty string since we're using predefined examples
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot <= len(FEWSHOT_ITEMS), f"Fewshot larger than {len(FEWSHOT_ITEMS)} is not supported for GSM8K"
+        super().__init__(num_fewshot)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        # Remove the bracketed computations from the question
+        question = re.sub(r"<<.*?>>", "", item["question"])
+        return f"Question: {question}\nAnswer:"
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        """Override to use predefined fewshot examples instead of sampling from dataset"""
+        return FEWSHOT_ITEMS[: self.num_fewshot]

eval_framework/tasks/benchmarks/hellaswag.py

@@ -0,0 +1,44 @@
+import re
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class HELLASWAG(BaseTask[str]):
+    """Hellaswag dataset: https://huggingface.co/datasets/Rowan/hellaswag
+    available data set sections: train, validation, test"""
+
+    NAME = "HellaSwag"
+    DATASET_PATH = "Rowan/hellaswag"
+    SAMPLE_SPLIT = "validation"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = Language.ENG
+
+    @staticmethod
+    def _preprocess(prompt: str) -> str:
+        # remove bracketed text
+        prompt = prompt.strip()
+        prompt = prompt.replace(" [title]", ". ")
+        prompt = re.sub("\\[.*?\\]", "", prompt)
+        prompt = prompt.replace("  ", " ")
+        return prompt
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        subject = self._preprocess(item["activity_label"])
+        question = self._preprocess(item["ctx_a"] + " " + item["ctx_b"].capitalize()).strip()
+        return f"{subject}: {question}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        ground_truth_index = int(item["label"] if item["label"] != "" else 0)
+        choices = [self._preprocess(ending) for ending in item["endings"]]
+        return f" {choices[ground_truth_index]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {self._preprocess(ending)}" for ending in item["endings"]]

eval_framework/tasks/benchmarks/hellaswag_de.py

@@ -0,0 +1,52 @@
+import re
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class HELLASWAG_DE(BaseTask[str]):
+    """Hellaswag dataset: https://huggingface.co/datasets/LeoLM/HellaSwag_de
+    available data set sections: train (1k rows), validation (10k rows)"""
+
+    NAME = "HellaSwag German"
+    DATASET_PATH = "LeoLM/HellaSwag_de"
+    SAMPLE_SPLIT = "validation"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = Language.DEU
+
+    @staticmethod
+    def _preprocess(prompt: str) -> str:
+        # remove bracketed text
+        prompt = prompt.strip()
+        prompt = prompt.replace(" [title]", ". ")
+        prompt = re.sub("\\[.*?\\]", "", prompt)
+        prompt = prompt.replace("  ", " ")
+        return prompt
+
+    def _load_dataset(self, subject: str) -> None:
+        super()._load_dataset(subject)
+        new_dataset = {}
+        for split, items in self.dataset.items():
+            # in the valid split, only 10035 out of 10042 items are well translated
+            new_dataset[split] = [item for item in items if len(item["endings_de"]) == len(item["endings"])]
+        self.dataset = new_dataset
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        subject = self._preprocess(item["activity_label_de"])
+        question = self._preprocess(item["ctx_de"]).strip()
+        return f"{subject}: {question}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        ground_truth_index = int(item["label"] if item["label"] != "" else 0)
+        choices = [self._preprocess(ending) for ending in item["endings_de"]]
+        return f" {choices[ground_truth_index]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {self._preprocess(ending)}" for ending in item["endings_de"]]

eval_framework/tasks/benchmarks/humaneval.py

@@ -0,0 +1,97 @@
+from typing import Any
+
+from eval_framework.metrics.completion.code_assertion import CodeCompletionAssertion
+from eval_framework.shared.types import BaseMetricContext
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType, Sample
+
+CODE_TO_EXECUTE = """
+{start_of_code}
+{completion_text}
+{test_code}
+try:
+  check({entry_point})
+  print(True)
+except Exception as e:
+  print(e)
+  print(False)
+"""
+
+
+class HumanEvalMetricContext(BaseMetricContext):
+    test: str
+    entry_point: str
+    prompt: str
+
+
+class HumanEval(BaseTask[str]):
+    """HumanEval dataset: https://huggingface.co/datasets/openai/openai_humaneval/"""
+
+    NAME = "Human Eval"
+    DATASET_PATH = "openai/openai_humaneval"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"  # (there is no dedicated split, few-shot is not expected for this dataset)
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [CodeCompletionAssertion]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+        self.stop_sequences: list[str] = ["```"]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"```python\n{item['prompt'].lstrip()}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return "Success"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return item["canonical_solution"]
+
+    def _get_context(self, item: dict[str, Any]) -> HumanEvalMetricContext:
+        return HumanEvalMetricContext(
+            test=item["test"],
+            entry_point=item["entry_point"],
+            prompt=item["prompt"],
+        )
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        assert sample is not None and sample.context is not None
+        assert isinstance(sample.context, HumanEvalMetricContext), "Expected HumanEvalMetricContext"
+        context = sample.context
+
+        for stop_sequence in self.stop_sequences:
+            if stop_sequence in completion_text:
+                completion_text = completion_text.split(stop_sequence)[0]
+
+        entry_point = context.entry_point
+        test_code = context.test
+        start_of_code = context.prompt
+        formatted_code = CODE_TO_EXECUTE.format(
+            start_of_code=start_of_code,
+            completion_text=completion_text,
+            test_code=test_code,
+            entry_point=entry_point,
+        )
+
+        return formatted_code
+
+
+class HumanEvalInstruct(HumanEval):
+    # See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/humaneval/humaneval_instruct.yaml
+    NAME = "Human Eval Instruct"
+    CUE_PREFIX = "Here is the completed function:\n```python\n"
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot == 0, "Fewshot is not supported for Human Eval Instruct"
+        super().__init__(num_fewshot)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        instruction_text = (
+            "Write a solution to the following problem and make sure that "
+            f"it passes the tests:\n```python\n{item['prompt'].lstrip()}"
+        )
+        return instruction_text
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return self.CUE_PREFIX + item["prompt"].lstrip()

eval_framework/tasks/benchmarks/ifeval.py

@@ -0,0 +1,78 @@
+from typing import Any
+
+from eval_framework.metrics.completion.ifeval import IFEvalMetric, IFEvalMetricContext
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class IFEval(BaseTask[str]):
+    """IFEval: Instruction Following Eval (https://arxiv.org/pdf/2311.07911)."""
+
+    NAME = "IFEval"
+    DATASET_PATH = "google/IFEval"
+    SAMPLE_SPLIT = "train"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [IFEvalMetric]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = {NO_SUBJECT: Language.ENG}
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+        assert num_fewshot == 0, "IFEval does not support few-shot prompting."
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return item["prompt"]
+
+    def _get_context(self, item: dict[str, Any]) -> IFEvalMetricContext:
+        assert "key" in item, "Expected 'key' in item"
+        assert "instruction_id_list" in item, "Expected 'instruction_id_list' in item"
+        assert "prompt" in item, "Expected 'prompt' in item"
+        assert "kwargs" in item, "Expected 'kwargs' in item"
+
+        new_kwargs = []
+        for d in item["kwargs"]:
+            # fixing undesired float fields in the dataset
+            assert all([abs(v - float(v)) < 1e-5 for v in d.values() if isinstance(v, float)])
+            new_kwargs.append({k: v if not isinstance(v, float) else int(v) for k, v in d.items()})
+
+        # fixing changes to the HF dataset done on Apr 10 2025
+        if item["key"] == 142:
+            new_kwargs[2]["relation"] = None
+            new_kwargs[2]["frequency"] = None
+            new_kwargs[2]["keywords"] = new_kwargs[2]["keyword"]
+            del new_kwargs[2]["keyword"]
+        if item["key"] == 1512:
+            new_kwargs[0]["relation"] = None
+
+        item["kwargs"] = new_kwargs
+
+        return IFEvalMetricContext(
+            key=item["key"],
+            instruction_id_list=item["instruction_id_list"],
+            prompt=item["prompt"],
+            additional_kwargs=item["kwargs"],
+        )
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return None
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return []
+
+
+class IFEvalFiSv(IFEval):
+    """Machine translated versions of the Instruction Following Evaluation (IFEval) benchmark."""
+
+    NAME = "IFEval Finnish & Swedish"
+    DATASET_PATH = "LumiOpen/ifeval_mt"
+    SUBJECTS = ["fi", "sv"]
+    LANGUAGE = {"fi": Language.FIN, "sv": Language.SWE}
+
+
+class IFEvalDe(IFEval):
+    """German version of the Instruction Following Evaluation (IFEval) benchmark."""
+
+    NAME = "IFEval German"
+    DATASET_PATH = "jzhang86/de_ifeval"
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = {NO_SUBJECT: Language.DEU}