eval-framework 0.2.7__py3-none-any.whl

eval_framework/tasks/benchmarks/gsm8k.py

@@ -0,0 +1,150 @@
+import re
+from typing import Any
+
+from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
+from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+
+ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)")
+
+# Predefined fewshot examples
+FEWSHOT_ITEMS = [
+    {
+        "question": (
+            "There are 15 trees in the grove. Grove workers will plant trees in the grove today. "
+            "After they are done, there will be 21 trees. "
+            "How many trees did the grove workers plant today?"
+        ),
+        "answer": (
+            "There are 15 trees originally. Then there were 21 trees after some more were planted. "
+            "So there must have been 21 - 15 = 6.\n#### 6"
+        ),
+    },
+    {
+        "question": (
+            "If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?"
+        ),
+        "answer": "There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5.\n#### 5",
+    },
+    {
+        "question": (
+            "Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?"
+        ),
+        "answer": (
+            "Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. "
+            "After eating 35, they had 74 - 35 = 39.\n#### 39"
+        ),
+    },
+    {
+        "question": (
+            "Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. "
+            "How many lollipops did Jason give to Denny?"
+        ),
+        "answer": (
+            "Jason started with 20 lollipops. Then he had 12 after giving some to Denny. "
+            "So he gave Denny 20 - 12 = 8.\n#### 8"
+        ),
+    },
+    {
+        "question": (
+            "Shawn has five toys. For Christmas, he got two toys each from his mom and dad. "
+            "How many toys does he have now?"
+        ),
+        "answer": (
+            "Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. "
+            "5 + 4 = 9.\n#### 9"
+        ),
+    },
+    {
+        "question": (
+            "There were nine computers in the server room. Five more computers were installed each day, "
+            "from monday to thursday. "
+            "How many computers are now in the server room?"
+        ),
+        "answer": (
+            "There were originally 9 computers. For each of 4 days, 5 more computers were "
+            "added. So 5 * 4 = 20 computers were added. 9 + 20 is 29.\n#### 29"
+        ),
+    },
+    {
+        "question": (
+            "Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. "
+            "How many golf balls did he have at the end of wednesday?"
+        ),
+        "answer": (
+            "Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. "
+            "After losing 2 more, he had 35 - 2 = 33 golf balls.\n#### 33"
+        ),
+    },
+    {
+        "question": "Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
+        "answer": (
+            "Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. "
+            "So she has 23 - 15 dollars left. 23 - 15 is 8.\n#### 8"
+        ),
+    },
+]
+
+
+class GSM8KEvalHarness(BaseTask[str]):
+    """GSM8K dataset: https://huggingface.co/datasets/openai/gsm8k
+    This version uses samples from the train split as fewshot examples.
+    """
+
+    NAME = "GSM8KEvalHarness"
+    DATASET_PATH = "gsm8k"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [AccuracyCompletion]
+    SUBJECTS = ["main"]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        # until: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/gsm8k/gsm8k.yaml
+        self.stop_sequences: list[str] = ["Question:"]
+        self.max_tokens = 1600
+
+    def _extract_answer(self, completion: str) -> str:
+        match = ANS_RE.search(completion)
+        if match:
+            match_str = match.group(1).strip()
+            match_str = match_str.replace(",", "")
+            return match_str
+        else:
+            return "[invalid]"
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        for stop_sequence in self.stop_sequences:
+            if stop_sequence in completion_text:
+                completion_text = completion_text.split(stop_sequence)[0]
+        return self._extract_answer(completion_text)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"Question: {item['question']}\nAnswer:"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return f" {item['answer']}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return self._extract_answer(item["answer"])
+
+
+class GSM8K(GSM8KEvalHarness):
+    NAME = "GSM8K"
+    FEWSHOT_SPLIT = ""  # Changed to empty string since we're using predefined examples
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot <= len(FEWSHOT_ITEMS), f"Fewshot larger than {len(FEWSHOT_ITEMS)} is not supported for GSM8K"
+        super().__init__(num_fewshot)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        # Remove the bracketed computations from the question
+        question = re.sub(r"<<.*?>>", "", item["question"])
+        return f"Question: {question}\nAnswer:"
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        """Override to use predefined fewshot examples instead of sampling from dataset"""
+        return FEWSHOT_ITEMS[: self.num_fewshot]

eval_framework/tasks/benchmarks/hellaswag.py

@@ -0,0 +1,69 @@
+import re
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import ConfidenceWeightedAccuracy
+from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
+from eval_framework.metrics.loglikelihood.ternary import TernaryScore
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class HELLASWAG(BaseTask[str]):
+    """Hellaswag dataset: https://huggingface.co/datasets/Rowan/hellaswag
+    available data set sections: train, validation, test"""
+
+    NAME = "HellaSwag"
+    DATASET_PATH = "Rowan/hellaswag"
+    SAMPLE_SPLIT = "validation"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = Language.ENG
+
+    @staticmethod
+    def _preprocess(prompt: str) -> str:
+        # remove bracketed text
+        prompt = prompt.strip()
+        prompt = prompt.replace(" [title]", ". ")
+        prompt = re.sub("\\[.*?\\]", "", prompt)
+        prompt = prompt.replace("  ", " ")
+        return prompt
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        subject = self._preprocess(item["activity_label"])
+        question = self._preprocess(item["ctx_a"] + " " + item["ctx_b"].capitalize()).strip()
+        return f"{subject}: {question}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        ground_truth_index = int(item["label"] if item["label"] != "" else 0)
+        choices = [self._preprocess(ending) for ending in item["endings"]]
+        return f" {choices[ground_truth_index]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {self._preprocess(ending)}" for ending in item["endings"]]
+
+
+class HELLASWAG_IDK(HELLASWAG):
+    NAME = "HellaSwag_IDK"
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        ConfidenceWeightedAccuracy,
+        DistributionalCorrectnessScore,
+        TernaryScore,
+    ]
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return (
+            "Complete the sentence only if you are confident, since mistakes may be penalised, while correct "
+            "completions receive points. It is acceptable to answer with 'I do not know' if you are unsure, "
+            "and you will receive 0 points."
+        )
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        completions = super()._get_possible_completions(item)
+        return (completions or []) + [" I do not know."]

eval_framework/tasks/benchmarks/hellaswag_de.py

@@ -0,0 +1,52 @@
+import re
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class HELLASWAG_DE(BaseTask[str]):
+    """Hellaswag dataset: https://huggingface.co/datasets/LeoLM/HellaSwag_de
+    available data set sections: train (1k rows), validation (10k rows)"""
+
+    NAME = "HellaSwag German"
+    DATASET_PATH = "LeoLM/HellaSwag_de"
+    SAMPLE_SPLIT = "validation"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = Language.DEU
+
+    @staticmethod
+    def _preprocess(prompt: str) -> str:
+        # remove bracketed text
+        prompt = prompt.strip()
+        prompt = prompt.replace(" [title]", ". ")
+        prompt = re.sub("\\[.*?\\]", "", prompt)
+        prompt = prompt.replace("  ", " ")
+        return prompt
+
+    def _load_dataset(self, subject: str) -> None:
+        super()._load_dataset(subject)
+        new_dataset = {}
+        for split, items in self.dataset.items():
+            # in the valid split, only 10035 out of 10042 items are well translated
+            new_dataset[split] = [item for item in items if len(item["endings_de"]) == len(item["endings"])]
+        self.dataset = new_dataset
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        subject = self._preprocess(item["activity_label_de"])
+        question = self._preprocess(item["ctx_de"]).strip()
+        return f"{subject}: {question}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        ground_truth_index = int(item["label"] if item["label"] != "" else 0)
+        choices = [self._preprocess(ending) for ending in item["endings_de"]]
+        return f" {choices[ground_truth_index]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {self._preprocess(ending)}" for ending in item["endings_de"]]

eval_framework/tasks/benchmarks/humaneval.py

@@ -0,0 +1,97 @@
+from typing import Any
+
+from eval_framework.metrics.completion.code_assertion import CodeCompletionAssertion
+from eval_framework.shared.types import BaseMetricContext
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType, Sample
+
+CODE_TO_EXECUTE = """
+{start_of_code}
+{completion_text}
+{test_code}
+try:
+  check({entry_point})
+  print(True)
+except Exception as e:
+  print(e)
+  print(False)
+"""
+
+
+class HumanEvalMetricContext(BaseMetricContext):
+    test: str
+    entry_point: str
+    prompt: str
+
+
+class HumanEval(BaseTask[str]):
+    """HumanEval dataset: https://huggingface.co/datasets/openai/openai_humaneval/"""
+
+    NAME = "Human Eval"
+    DATASET_PATH = "openai/openai_humaneval"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"  # (there is no dedicated split, few-shot is not expected for this dataset)
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [CodeCompletionAssertion]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+        self.stop_sequences: list[str] = ["```"]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"```python\n{item['prompt'].lstrip()}"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return "Success"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return item["canonical_solution"]
+
+    def _get_context(self, item: dict[str, Any]) -> HumanEvalMetricContext:
+        return HumanEvalMetricContext(
+            test=item["test"],
+            entry_point=item["entry_point"],
+            prompt=item["prompt"],
+        )
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        assert sample is not None and sample.context is not None
+        assert isinstance(sample.context, HumanEvalMetricContext), "Expected HumanEvalMetricContext"
+        context = sample.context
+
+        for stop_sequence in self.stop_sequences:
+            if stop_sequence in completion_text:
+                completion_text = completion_text.split(stop_sequence)[0]
+
+        entry_point = context.entry_point
+        test_code = context.test
+        start_of_code = context.prompt
+        formatted_code = CODE_TO_EXECUTE.format(
+            start_of_code=start_of_code,
+            completion_text=completion_text,
+            test_code=test_code,
+            entry_point=entry_point,
+        )
+
+        return formatted_code
+
+
+class HumanEvalInstruct(HumanEval):
+    # See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/humaneval/humaneval_instruct.yaml
+    NAME = "Human Eval Instruct"
+    CUE_PREFIX = "Here is the completed function:\n```python\n"
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot == 0, "Fewshot is not supported for Human Eval Instruct"
+        super().__init__(num_fewshot)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        instruction_text = (
+            "Write a solution to the following problem and make sure that "
+            f"it passes the tests:\n```python\n{item['prompt'].lstrip()}"
+        )
+        return instruction_text
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return self.CUE_PREFIX + item["prompt"].lstrip()

eval_framework/tasks/benchmarks/ifeval.py

@@ -0,0 +1,78 @@
+from typing import Any
+
+from eval_framework.metrics.completion.ifeval import IFEvalMetric, IFEvalMetricContext
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class IFEval(BaseTask[str]):
+    """IFEval: Instruction Following Eval (https://arxiv.org/pdf/2311.07911)."""
+
+    NAME = "IFEval"
+    DATASET_PATH = "google/IFEval"
+    SAMPLE_SPLIT = "train"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [IFEvalMetric]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = {NO_SUBJECT: Language.ENG}
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+        assert num_fewshot == 0, "IFEval does not support few-shot prompting."
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return item["prompt"]
+
+    def _get_context(self, item: dict[str, Any]) -> IFEvalMetricContext:
+        assert "key" in item, "Expected 'key' in item"
+        assert "instruction_id_list" in item, "Expected 'instruction_id_list' in item"
+        assert "prompt" in item, "Expected 'prompt' in item"
+        assert "kwargs" in item, "Expected 'kwargs' in item"
+
+        new_kwargs = []
+        for d in item["kwargs"]:
+            # fixing undesired float fields in the dataset
+            assert all([abs(v - float(v)) < 1e-5 for v in d.values() if isinstance(v, float)])
+            new_kwargs.append({k: v if not isinstance(v, float) else int(v) for k, v in d.items()})
+
+        # fixing changes to the HF dataset done on Apr 10 2025
+        if item["key"] == 142:
+            new_kwargs[2]["relation"] = None
+            new_kwargs[2]["frequency"] = None
+            new_kwargs[2]["keywords"] = new_kwargs[2]["keyword"]
+            del new_kwargs[2]["keyword"]
+        if item["key"] == 1512:
+            new_kwargs[0]["relation"] = None
+
+        item["kwargs"] = new_kwargs
+
+        return IFEvalMetricContext(
+            key=item["key"],
+            instruction_id_list=item["instruction_id_list"],
+            prompt=item["prompt"],
+            additional_kwargs=item["kwargs"],
+        )
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return None
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return []
+
+
+class IFEvalFiSv(IFEval):
+    """Machine translated versions of the Instruction Following Evaluation (IFEval) benchmark."""
+
+    NAME = "IFEval Finnish & Swedish"
+    DATASET_PATH = "LumiOpen/ifeval_mt"
+    SUBJECTS = ["fi", "sv"]
+    LANGUAGE = {"fi": Language.FIN, "sv": Language.SWE}
+
+
+class IFEvalDe(IFEval):
+    """German version of the Instruction Following Evaluation (IFEval) benchmark."""
+
+    NAME = "IFEval German"
+    DATASET_PATH = "jzhang86/de_ifeval"
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = {NO_SUBJECT: Language.DEU}

eval_framework/tasks/benchmarks/include.py

@@ -0,0 +1,119 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.utils import get_n_letters
+
+INCLUDE_SUBJECTS = [
+    "Albanian",
+    "Arabic",
+    "Armenian",
+    "Azerbaijani",
+    "Basque",
+    "Belarusian",
+    "Bengali",
+    "Bulgarian",
+    "Chinese",
+    "Croatian",
+    "Dutch",
+    "Estonian",
+    "Finnish",
+    "French",
+    "Georgian",
+    "German",
+    "Greek",
+    "Hebrew",
+    "Hindi",
+    "Hungarian",
+    "Indonesian",
+    "Italian",
+    "Japanese",
+    "Kazakh",
+    "Korean",
+    "Lithuanian",
+    "Malay",
+    "Malayalam",
+    "Nepali",
+    "North Macedonian",
+    "Persian",
+    "Polish",
+    "Portuguese",
+    "Russian",
+    "Serbian",
+    "Spanish",
+    "Tagalog",
+    "Tamil",
+    "Telugu",
+    "Turkish",
+    "Ukrainian",
+    "Urdu",
+    "Uzbek",
+    "Vietnamese",
+]
+
+
+def subject_to_language(subject: str) -> Language:
+    if subject == "Greek":
+        return Language.ELL  # type: ignore[attr-defined]
+    elif subject == "Malay":
+        return Language.MSA  # type: ignore[attr-defined]
+    elif subject == "Nepali":
+        return Language.NEP  # type: ignore[attr-defined]
+    elif subject == "North Macedonian":
+        return Language.MKD  # type: ignore[attr-defined]
+    elif subject == "Croatian":
+        return Language.HRV  # type: ignore[attr-defined]
+    elif subject == "Serbian":
+        return Language.SRP  # type: ignore[attr-defined]
+    else:
+        return Language(subject)
+
+
+class INCLUDE(BaseTask[str]):
+    """INCLUDE dataset: https://huggingface.co/datasets/CohereLabs/include-base-44"""
+
+    NAME = "INCLUDE"
+    DATASET_PATH = "CohereLabs/include-base-44"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "validation"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = INCLUDE_SUBJECTS
+    LANGUAGE = {lang: subject_to_language(lang) for lang in INCLUDE_SUBJECTS}
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        self.keys = get_n_letters(4)
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return f"The following are multiple choice questions (with answers) in {item['language']}."  # noqa: E501
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        question = item["question"].strip()
+        choices = "".join(
+            [
+                f"{key}. {choice}\n"
+                for key, choice in zip(
+                    self.keys, [item["option_a"], item["option_b"], item["option_c"], item["option_d"]]
+                )
+            ]
+        )
+        return f"Question: {question}\n{choices}"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return f" {self.keys[item['answer']]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {key}" for key in self.keys]