eval-framework 0.2.0__py3-none-any.whl

eval_framework/tasks/benchmarks/opengptx_eu20.py

@@ -0,0 +1,363 @@
+import random
+import re
+from typing import Any
+
+from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, Language, SubjectType
+from eval_framework.tasks.benchmarks.arc import ARC
+from eval_framework.tasks.benchmarks.gsm8k import GSM8K
+from eval_framework.tasks.benchmarks.hellaswag import HELLASWAG
+from eval_framework.tasks.benchmarks.mmlu import MMLU, MMLU_SUBJECTS
+from eval_framework.tasks.benchmarks.mmlu_de import MMLU_SUBJECTS_TRANSLATION
+from eval_framework.tasks.benchmarks.truthfulqa import TRUTHFULQA
+
+
+class ARC_EU20_DE(ARC):
+    """
+    EU20 Benchmarks from the openGPT-X paper:
+    - https://arxiv.org/abs/2410.08928
+    - leaderboard: https://huggingface.co/spaces/openGPT-X/european-llm-leaderboard
+
+
+    https://huggingface.co/datasets/openGPT-X/arcx
+      entries in 'challenge_DE': 1172 test, 299 validation, 198 train
+      entries in 'easy_DE': 2376 test, 570 validation, 197 train
+            features: ['id', 'question', 'choices', 'answerKey'],
+      SUBJECTS = ['challenge_BG', 'easy_BG', 'challenge_DA', 'easy_DA', 'challenge_DE', 'easy_DE', 'challenge_ET', 'easy_ET', 'challenge_FI', 'easy_FI', 'challenge_FR', 'easy_FR', 'challenge_EL', 'easy_EL', 'challenge_IT', 'easy_IT', 'challenge_LV', 'easy_LV', 'challenge_LT', 'easy_LT', 'challenge_NL', 'easy_NL', 'challenge_PL', 'easy_PL', 'challenge_PT-PT', 'easy_PT-PT', 'challenge_RO', 'easy_RO', 'challenge_SV', 'easy_SV', 'challenge_SK', 'easy_SK', 'challenge_SL', 'easy_SL', 'challenge_ES', 'easy_ES', 'challenge_CS', 'easy_CS', 'challenge_HU', 'easy_HU']
+    """  # noqa: E501
+
+    NAME = "ARC_EU20_DE"
+    DATASET_PATH = "openGPT-X/arcx"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    SUBJECTS = ["challenge_DE", "easy_DE"]
+    LANGUAGE = Language.DEU
+
+
+class ARC_EU20_FR(ARC):
+    NAME = "ARC_EU20_FR"
+    DATASET_PATH = "openGPT-X/arcx"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    SUBJECTS = ["challenge_FR", "easy_FR"]
+    LANGUAGE = Language.FRA
+
+
+class GSM8K_EU20_DE(GSM8K):
+    """
+    https://huggingface.co/datasets/openGPT-X/gsm8kx
+      entries in 'DE': 1319 test, 104 train
+            features: ['question', 'answer', 'id'],
+      SUBJECTS = ['BG', 'DA', 'DE', 'ET', 'FI', 'FR', 'EL', 'IT', 'LV', 'LT', 'NL', 'PL', 'PT-PT', 'RO', 'SV', 'SK', 'SL', 'ES', 'CS', 'HU']
+    """  # noqa: E501
+
+    NAME = "GSM8K_EU20_DE"
+    DATASET_PATH = "openGPT-X/gsm8kx"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    SUBJECTS = ["DE"]
+    LANGUAGE = Language.DEU
+
+
+class GSM8K_EU20_FR(GSM8K):
+    NAME = "GSM8K_EU20_FR"
+    DATASET_PATH = "openGPT-X/gsm8kx"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    SUBJECTS = ["FR"]
+    LANGUAGE = Language.FRA
+
+
+class HELLASWAG_EU20_DE(HELLASWAG):
+    """
+    https://huggingface.co/datasets/openGPT-X/hellaswagx
+      entries in 'DE': 99 train, 9979 validation
+            features: ['ind', 'activity_label', 'ctx_a', 'ctx_b', 'ctx', 'endings', 'source_id', 'split', 'split_type', 'label'],
+      SUBJECTS = ['BG', 'DA', 'DE', 'ET', 'FI', 'FR', 'EL', 'IT', 'LV', 'LT', 'NL', 'PL', 'PT-PT', 'RO', 'SV', 'SK', 'SL', 'ES', 'CS', 'HU']
+    """  # noqa: E501
+
+    NAME = "HellaSwag_EU20_DE"
+    DATASET_PATH = "openGPT-X/hellaswagx"
+    SAMPLE_SPLIT = "train"
+    FEWSHOT_SPLIT = "validation"
+    SUBJECTS = ["DE"]
+    LANGUAGE = Language.DEU
+
+
+class HELLASWAG_EU20_FR(HELLASWAG):
+    NAME = "HellaSwag_EU20_FR"
+    DATASET_PATH = "openGPT-X/hellaswagx"
+    SAMPLE_SPLIT = "train"
+    FEWSHOT_SPLIT = "validation"
+    SUBJECTS = ["FR"]
+    LANGUAGE = Language.FRA
+
+
+TRUTHFULQA_EU20_DE_FEWSHOT_ITEMS = [
+    {
+        "question": "Was ist die Lebenserwartung in den Vereinigten Staaten?",
+        "response": "Die Lebenserwartung in den Vereinigten Staaten beträgt 78 Jahre.",
+    },
+    {
+        "question": "Wer war 1955 Präsident der Vereinigten Staaten?",
+        "response": "Dwight D. Eisenhower war 1955 Präsident der Vereinigten Staaten.",
+    },
+    {"question": "Welcher Partei gehörte er an?", "response": "Er gehörte der Republikanischen Partei an."},
+    {"question": "Was ist die Quadratwurzel von Banane?", "response": "Ich habe keinen Kommentar."},
+    {
+        "question": "Wie funktioniert ein Teleskop?",
+        "response": (
+            "Teleskope verwenden Linsen oder Spiegel, um Licht zu fokussieren und Objekte näher erscheinen zu lassen."
+        ),
+    },
+    {
+        "question": "Wo fanden die Olympischen Spiele 1992 statt?",
+        "response": "Die Olympischen Spiele 1992 fanden in Barcelona, Spanien, statt.",
+    },
+]
+
+
+class TRUTHFULQA_EU20_DE(TRUTHFULQA):
+    """
+    https://huggingface.co/datasets/openGPT-X/truthfulqax
+      entries in 'mc_DE': 817 validation
+            features: ['question', 'mc1_targets', 'mc2_targets', 'id'],
+      entries in 'gen_DE': 817 validation
+            features: ['type', 'category', 'question', 'best_answer', 'correct_answers', 'incorrect_answers', 'source', 'id'],
+      SUBJECTS = ['mc_BG', 'gen_BG', 'mc_DA', 'gen_DA', 'mc_DE', 'gen_DE', 'mc_ET', 'gen_ET', 'mc_FI', 'gen_FI', 'mc_FR', 'gen_FR', 'mc_EL', 'gen_EL', 'mc_IT', 'gen_IT', 'mc_LV', 'gen_LV', 'mc_LT', 'gen_LT', 'mc_NL', 'gen_NL', 'mc_PL', 'gen_PL', 'mc_PT-PT', 'gen_PT-PT', 'mc_RO', 'gen_RO', 'mc_SV', 'gen_SV', 'mc_SK', 'gen_SK', 'mc_SL', 'gen_SL', 'mc_ES', 'gen_ES', 'mc_CS', 'gen_CS', 'mc_HU', 'gen_HU']
+    """  # noqa: E501
+
+    NAME = "TruthfulQA_EU20_DE"
+    DATASET_PATH = "openGPT-X/truthfulqax"
+    LANGUAGE = Language.DEU
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return TRUTHFULQA_EU20_DE_FEWSHOT_ITEMS[: self.num_fewshot]
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        """The original dataset only provides one subject 'multiple_choice', but with multiple target columns
+        this should be seen as multiple subjects.
+        Alternatively we would need to adjust the dataset and upload it with propper
+        subject names to huggingface."""
+
+        self.target_identifier = f"{str(subject)}_targets"
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name="mc_DE")
+        self.dataset = {}
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            if split not in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                continue
+
+            data_list = list(data)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            self.dataset[split] = data_list
+
+
+# translation of fewshot examples from Appendix E in https://arxiv.org/pdf/2109.07958
+TRUTHFULQA_EU20_FR_FEWSHOT_ITEMS = [
+    {
+        "question": "Quelle est l'espérance de vie humaine aux États-Unis ?",
+        "response": "L'espérance de vie humaine aux États-Unis est de 78 ans.",
+    },
+    {
+        "question": "Qui était président des États-Unis en 1955 ?",
+        "response": "Dwight D. Eisenhower était président des États-Unis en 1955.",
+    },
+    {"question": "À quel parti appartenait-il ?", "response": "Il appartenait au Parti républicain."},
+    {"question": "Quelle est la racine carrée de la banane ?", "response": "Je n'ai aucun commentaire."},
+    {
+        "question": "Comment fonctionne un télescope ?",
+        "response": "Les télescopes utilisent des lentilles ou des miroirs pour focaliser la lumière et faire apparaître les objets plus proches.",  # noqa: E501
+    },
+    {
+        "question": "Où se sont déroulés les Jeux olympiques de 1992 ?",
+        "response": "Les Jeux olympiques de 1992 se sont déroulés à Barcelone, en Espagne.",
+    },
+]
+
+
+class TRUTHFULQA_EU20_FR(TRUTHFULQA):
+    NAME = "TruthfulQA_EU20_FR"
+    DATASET_PATH = "openGPT-X/truthfulqax"
+    LANGUAGE = Language.FRA
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        self.target_identifier = f"{str(subject)}_targets"
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name="mc_FR")
+        self.dataset = {}
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            if split not in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                continue
+
+            data_list = list(data)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            self.dataset[split] = data_list
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return TRUTHFULQA_EU20_FR_FEWSHOT_ITEMS[: self.num_fewshot]
+
+
+class MMLU_EU20_DE(MMLU):
+    """
+    https://huggingface.co/datasets/openGPT-X/mmlux
+      entries in 'philosophy_DE': 311 test, 5 dev, 5 validation
+           features: ['question', 'choices', 'answer', 'id'],
+    """
+
+    NAME = "MMLU_EU20_DE"
+    DATASET_PATH = "openGPT-X/mmlux"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "dev"  # one could merge dev and validation to have a larger pool of fewshot examples
+    SUBJECTS = [i + "_DE" for i in MMLU_SUBJECTS]
+    PERTURBATION_UNMODIFIABLE_WORDS = MMLU.PERTURBATION_UNMODIFIABLE_WORDS + ["Frage"]
+    LANGUAGE = Language.DEU
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        name = subject if subject != NO_SUBJECT else None
+
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name=name)
+        self.dataset = {}
+
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            data_list = []
+            for item in data:
+                item["subject"] = subject
+                data_list.append(item)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            if split in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                self.dataset[split] = data_list
+
+    def _get_subject_name(self, item: dict[str, Any]) -> str:
+        # removing DE suffix
+        subject = re.sub(r"_DE$", "", item["subject"])
+        return MMLU_SUBJECTS_TRANSLATION[subject]
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return f"Die folgenden sind Multiple Choice Fragen (mit Antworten) über {self._get_subject_name(item)}."  # noqa: E501
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        question = item["question"].strip()
+        choices = "".join([f"{key}. {choice}\n" for key, choice in zip(self.keys, item["choices"])])
+        return f"Frage: {question}\n{choices}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Antwort:"
+
+
+MMLU_SUBJECTS_TRANSLATION_FR = {
+    "abstract_algebra": "Algèbre Abstraite",
+    "anatomy": "Anatomie",
+    "astronomy": "Astronomie",
+    "business_ethics": "Éthique des Affaires",
+    "clinical_knowledge": "Connaissances Cliniques",
+    "college_biology": "Biologie Universitaire",
+    "college_chemistry": "Chimie Universitaire",
+    "college_computer_science": "Informatique Universitaire",
+    "college_mathematics": "Mathématiques Universitaires",
+    "college_medicine": "Médecine Universitaire",
+    "college_physics": "Physique Universitaire",
+    "computer_security": "Sécurité Informatique",
+    "conceptual_physics": "Physique Conceptuelle",
+    "econometrics": "Économétrie",
+    "electrical_engineering": "Génie Électrique",
+    "elementary_mathematics": "Mathématiques Élémentaires",
+    "formal_logic": "Logique Formelle",
+    "global_facts": "Faits Mondiaux",
+    "high_school_biology": "Biologie au Lycée",
+    "high_school_chemistry": "Chimie au Lycée",
+    "high_school_computer_science": "Informatique au Lycée",
+    "high_school_european_history": "Histoire Européenne au Lycée",
+    "high_school_geography": "Géographie au Lycée",
+    "high_school_government_and_politics": "Gouvernement et Politique au Lycée",
+    "high_school_macroeconomics": "Macroéconomie au Lycée",
+    "high_school_mathematics": "Mathématiques au Lycée",
+    "high_school_microeconomics": "Microéconomie au Lycée",
+    "high_school_physics": "Physique au Lycée",
+    "high_school_psychology": "Psychologie au Lycée",
+    "high_school_statistics": "Statistiques au Lycée",
+    "high_school_us_history": "Histoire des États-Unis au Lycée",
+    "high_school_world_history": "Histoire du Monde au Lycée",
+    "human_aging": "Vieillissement Humain",
+    "human_sexuality": "Sexualité Humaine",
+    "international_law": "Droit International",
+    "jurisprudence": "Jurisprudence",
+    "logical_fallacies": "Sophismes Logiques",
+    "machine_learning": "Apprentissage Automatique",
+    "management": "Gestion",
+    "marketing": "Marketing",
+    "medical_genetics": "Génétique Médicale",
+    "miscellaneous": "Divers",
+    "moral_disputes": "Conflits Moraux",
+    "moral_scenarios": "Scénarios Moraux",
+    "nutrition": "Nutrition",
+    "philosophy": "Philosophie",
+    "prehistory": "Préhistoire",
+    "professional_accounting": "Comptabilité Professionnelle",
+    "professional_law": "Droit Professionnel",
+    "professional_medicine": "Médecine Professionnelle",
+    "professional_psychology": "Psychologie Professionnelle",
+    "public_relations": "Relations Publiques",
+    "security_studies": "Études de Sécurité",
+    "sociology": "Sociologie",
+    "us_foreign_policy": "Politique Étrangère des États-Unis",
+    "virology": "Virologie",
+    "world_religions": "Religions du Monde",
+}
+
+
+class MMLU_EU20_FR(MMLU):
+    NAME = "MMLU_EU20_FR"
+    DATASET_PATH = "openGPT-X/mmlux"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "dev"
+    SUBJECTS = [i + "_FR" for i in MMLU_SUBJECTS]
+    LANGUAGE = Language.FRA
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        name = subject if subject != NO_SUBJECT else None
+
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name=name)
+        self.dataset = {}
+
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            data_list = []
+            for item in data:
+                item["subject"] = subject
+                data_list.append(item)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            if split in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                self.dataset[split] = data_list
+
+    def _get_subject_name(self, item: dict[str, Any]) -> str:
+        # removing FR suffix
+        subject = re.sub(r"_FR$", "", item["subject"])
+        return MMLU_SUBJECTS_TRANSLATION_FR[subject]
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return f"Les questions suivantes sont des questions à choix multiples (avec réponses) sur {self._get_subject_name(item)}."  # noqa: E501
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        question = item["question"].strip()
+        choices = "".join([f"{key}. {choice}\n" for key, choice in zip(self.keys, item["choices"])])
+        return f"Question: {question}\n{choices}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Réponse:"

eval_framework/tasks/benchmarks/pawsx.py

@@ -0,0 +1,65 @@
+from typing import Any
+
+from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
+from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+
+
+class PAWSX(BaseTask[str]):
+    """PAWSX dataset: https://huggingface.co/datasets/google-research-datasets/paws-x
+    used in the way suggested in PARAPHRASUS benchmark (https://arxiv.org/pdf/2409.12060)."""
+
+    NAME = "PAWS-X"
+    DATASET_PATH = "google-research-datasets/paws-x"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "validation"
+    RESPONSE_TYPE = ResponseType.COMPLETION  # LOGLIKELIHOODS would also make sense but staying true to PARAPHRASUS
+    METRICS = [AccuracyCompletion]
+    SUBJECTS = ["en", "de"]  # ["es", "fr", "ja", "ko", "zh"] -- disabled as irrelevant for the time being
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Ja", "Nein", "Paraphrasen", "Yes", "No", "paraphrases"]
+    LANGUAGE = {"en": Language.ENG, "de": Language.DEU}
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        self.num_fewshot = num_fewshot
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        # PARAPHRASUS seems to use English prompt for all languages but that's a bit weird, let's do it properly.
+        match item["subject"]:
+            case "de":
+                return (
+                    "Sind die folgenden Sätze Paraphrasen?\n"
+                    f"Satz 1: {item['sentence1']}\n"
+                    f"Satz 2: {item['sentence2']}\n"
+                    "Antworte mit 'Ja' oder 'Nein'.\n"
+                )
+            case _:
+                # Please translate to other language as necessary
+                return (
+                    "Are the following sentences paraphrases?\n"
+                    f"Sentence 1: {item['sentence1']}\n"
+                    f"Sentence 2: {item['sentence2']}\n"
+                    "Answer with 'Yes' or 'No'.\n"
+                )
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        match item["subject"]:
+            case "de":
+                return "Ja" if item["label"] == "1" else "Nein"
+            case _:
+                # Please translate to other language as necessary
+                return "Yes" if item["label"] == "1" else "No"
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        return completion_text.strip().strip("\"'.")
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        # Note that this, together with BaseTask._get_messages(), produces a different prompt structure than
+        # what PARAPHRASUS suggests in Figure 4. But both seem approaches are somehow valid...
+        examples: list[dict] = []
+        for _ in range(1000):
+            example = self.rnd.choice(self.dataset[self.FEWSHOT_SPLIT])
+            # Ensure half of the examples is negative and half positive.
+            if example["label"] == (len(examples) % 2) and example not in examples:
+                examples.append(example)
+            if len(examples) >= self.num_fewshot:
+                break
+        return examples

eval_framework/tasks/benchmarks/piqa.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class PIQA(BaseTask[str]):
+    """PIQA dataset: https://huggingface.co/datasets/ybisk/piqa"""
+
+    NAME = "PIQA"
+    DATASET_PATH = "ybisk/piqa"
+    SAMPLE_SPLIT = "validation"  # 1838 examples (same split as lm-eval)
+    FEWSHOT_SPLIT = "test"  # 3084 examples
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question"]
+    LANGUAGE = Language.ENG
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"Question: {item['goal']}\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        truth = item["sol1"] if item["label"] == 0 else item["sol2"]
+        return f" {truth}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {choice}" for choice in [item["sol1"], item["sol2"]]]

eval_framework/tasks/benchmarks/quality.py

@@ -0,0 +1,56 @@
+import random
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import RANDOM_SEED, BaseTask, Language, ResponseType, SubjectType
+
+
+class QUALITY(BaseTask[str]):
+    NAME = "QuALITY"
+    DATASET_PATH = "emozilla/quality"
+    SAMPLE_SPLIT = "validation"
+    FEWSHOT_SPLIT = "validation"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = ["hard", "easy"]
+
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Article", "Question", "Answer"]
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot == 0, "QuALITY only supports zero fewshot examples"
+        super().__init__(num_fewshot)
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH)
+        self.dataset = {}
+
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            if split not in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                continue
+
+            data_list = [item for item in data if item["hard"] == (subject == "hard")]
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            self.dataset[split] = data_list
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        article = item["article"]
+        question = item["question"]
+        return f"Article: {article}\nQuestion: {question}\n"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return f" {item['options'][item['answer']]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {option}" for option in item["options"]]

eval_framework/tasks/benchmarks/sciq.py

@@ -0,0 +1,44 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+
+
+class SCIQ(BaseTask[str]):
+    """SciQ dataset: https://huggingface.co/datasets/allenai/sciq"""
+
+    NAME = "SciQ"
+    DATASET_PATH = "allenai/sciq"
+    SAMPLE_SPLIT = "validation"  # 1000 examples (same split as lm-eval)
+    FEWSHOT_SPLIT = "test"  # 1000 examples
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question"]
+    LANGUAGE = Language.ENG
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"{item['support'].lstrip()}\nQuestion: {item['question']}\n"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return f" {item['correct_answer']}"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        choices = [
+            item["distractor1"],
+            item["distractor2"],
+            item["distractor3"],
+            item["correct_answer"],
+        ]
+        return [f" {choice}" for choice in choices]

eval_framework/tasks/benchmarks/sphyr.py

@@ -0,0 +1,75 @@
+from typing import Any
+
+from eval_framework.metrics.completion.grid_difference import GridDifference
+from eval_framework.tasks.base import BaseTask, Language, ResponseType
+
+SUBJECTS = [
+    "1_random_cell_easy",
+    "5_random_cell_easy",
+    "10_random_cell_easy",
+    "1_random_row_easy",
+    "3_random_row_easy",
+    "1_random_column_easy",
+    "3_random_column_easy",
+    "full_easy",
+    "1_random_cell_hard",
+    "5_random_cell_hard",
+    "10_random_cell_hard",
+    "1_random_row_hard",
+    "3_random_row_hard",
+    "1_random_column_hard",
+    "3_random_column_hard",
+    "full_hard",
+]
+
+SYSTEM_PROMPT = """You are given a structural material distribution represented as a grid. Each cell can have one of the following states:
+- 'L' indicates applied load.
+- 'V' indicates void.
+- 'S' indicates support.
+
+The goal is to predict the correct material distribution by filling in all {FILL_INSTRUCTION}, based on the surrounding structure and implicit physical reasoning (such as load paths, supports, and forces).
+
+Important: The completed structure should use as little material as possible while remaining stable and plausible for carrying the applied forces. Minimize material usage unless necessary for structural support."""  # noqa: E501
+
+PROMPT_TEMPLATE = """Below is the input grid with masked regions:
+
+{GRID}
+
+Please output the completed grid by replacing all {FILL_INSTRUCTION}.
+Maintain the same format as the input: one row per line, cells separated by spaces, and the total number of rows and columns unchanged.
+Return only the completed grid without any additional explanation."""  # noqa: E501
+
+EASY_FILL_INSTRUCTION = "'V' cells with either '1' (solid) or '0' (empty)"
+
+HARD_FILL_INSTRUCTION = (
+    "'V' cells with a floating point number between 0 and 1, with one decimal place (e.g., 0.0, 0.1, 0.2, ..., 1.0)"
+)
+
+
+class SPHYR(BaseTask[str]):
+    """SPhyR dataset: https://huggingface.co/datasets/philippds/SPhyR"""
+
+    NAME = "SPHYR"
+    DATASET_PATH = "philippds/SPhyR"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = ""
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [GridDifference]
+    SUBJECTS = SUBJECTS
+    PERTURBATION_UNMODIFIABLE_WORDS = None
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot == 0, "Fewshot is not supported for SPHYR"
+        super().__init__(num_fewshot)
+
+    def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
+        FILL_INSTRUCTION = EASY_FILL_INSTRUCTION if "easy" in item["subject"] else HARD_FILL_INSTRUCTION
+        return SYSTEM_PROMPT.format(FILL_INSTRUCTION=FILL_INSTRUCTION)
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        FILL_INSTRUCTION = EASY_FILL_INSTRUCTION if "easy" in item["subject"] else HARD_FILL_INSTRUCTION
+        return PROMPT_TEMPLATE.format(GRID=item["input_grid"], FILL_INSTRUCTION=FILL_INSTRUCTION)
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return item["ground_truth"]