crfm-helm 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/codeinsights_edge_case_scenario.py

@@ -0,0 +1,192 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsEdgeCaseScenario(Scenario):
+    name = "codeinsights_edge_case"
+    description = "Evaluate alignment in edge case failure between LLM-generated code and student code"
+    tags = ["codeinsights", "c++", "edge_case"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario5_data.csv")
+
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        # Load test cases (unit tests)
+        test_cases = self._load_test_cases()
+
+        # Get available question IDs with test cases
+        available_question_ids = set()
+        if test_cases:
+            available_question_ids = set(test_cases.keys())
+            print(f"Loaded test cases for {len(available_question_ids)} questions")
+        else:
+            print("WARNING: No test cases loaded!")
+            return []
+
+        instances = []
+        skipped_no_tests = 0
+        skipped_insufficient_data = 0
+
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            target = student_df.iloc[0]
+
+            # Check if target question has test cases BEFORE processing
+            target_question_id = target.get("question_unittest_id", None)
+            if not target_question_id or str(target_question_id) not in available_question_ids:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: No test cases available")
+                continue
+
+            # Get test cases for this question (we know they exist now)
+            target_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                target_test_cases.append(testcase)
+
+            # Verify test cases are not empty
+            if not tc_parsing_success:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: Empty test cases")
+                continue
+
+            if len(target_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                target_test_cases = target_test_cases[: self.num_testcases]
+
+            # Get student pass pattern for the target question
+            student_correctness_pattern = target.get("pass", None)
+            if student_correctness_pattern is not None:
+                main_part = int(student_correctness_pattern)
+                # Convert each character to an int
+                student_correctness_list = [int(ch) for ch in str(main_part)]
+            else:
+                student_correctness_list = []
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            print(f"\n=== ACCEPTED INSTANCE: Student {student_id}, Question {target_question_id} ===")
+            print(f"Test cases loaded: {len(target_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            prompt = (
+                "You are analyzing a student’s likely mistakes on an upcoming programming problem.\n"
+                "Your task: **predict exactly ONE unit-test index (0-based) that the student is most likely to fail.**\n"  # noqa: E501
+                "Return *only* that integer. No explanation.\n\n"
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "For the given programming question, identify which unit test the student would fail considering "
+                "their past performance, as well as consideration of unit test difficulty.\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Tests: {target_test_cases}\n"
+                if target_test_cases
+                else ""
+                "Think silently about:\n"
+                "• Which test seems hardest for the given topic?\n"
+                "• Where has the student historically struggled?\n"
+                "• Any edge-cases in the tests’ inputs/outputs?\n\n"
+                "******** ANSWER FORMAT (STRICT) ********\n"
+                "<integer>\n"
+                "****************************************"
+            )
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": target_test_cases,
+                        "question_id": str(target_question_id),
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+
+        # Print summary statistics
+        print("\n=== INSTANCE CREATION SUMMARY ===")
+        print(f"Skipped (insufficient data): {skipped_insufficient_data}")
+        print(f"Skipped (no test cases): {skipped_no_tests}")
+        print(f"Available test case question IDs: {len(available_question_ids)}")
+
+        if len(instances) >= 5:
+            print("Sample created instances:")
+            for i, inst in enumerate(instances[:5]):
+                if inst.extra_data is None:
+                    test_count = 0
+                else:
+                    test_count = len(inst.extra_data.get("test_cases", []))
+                print(f"  {inst.id}: {test_count} test cases")
+
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_student_coding_scenario.py

@@ -0,0 +1,162 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentCodingScenario(Scenario):
+    name = "codeinsights_student_coding"
+    description = "Mimic student C++ style on foundational questions"
+    tags = ["codeinsights", "c++", "student_coding"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student style, attempt this:\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py

@@ -0,0 +1,188 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentMistakeScenario(Scenario):
+    name = "codeinsights_student_mistake"
+    description = "Mimic how students mistake their C++ codes on foundational questions"
+    tags = ["codeinsights", "c++", "student_mistake"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario3_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values(by=["student_id", "question_unittest_id", "timestamp"])
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "When students submit a code to the platform, it will be tested by number of unit tests, where"
+                "- Unit test pass rate = proportion of unit tests passed with the code \n"
+                "- Full pass rate   = proportion of code passing all unit tests\n\n"
+                "=== Past Mistake Examples ===\n"
+                "Example 1 (Week {first['week']}, Topic: {first['topic']}):\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{first['response_mistake']}\n\n"
+                "Example 2 (Week {second['week']}, Topic: {second['topic']}):\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{second['response_mistake']}\n\n"
+                "Example 3 (Week {third['week']}, Topic: {third['topic']}):\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{third['response_mistake']}\n\n"
+                "=== New Target Problem ===\n"
+                f"Week: {target['week']}, Topic: {target['topic']}\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "⚠**Instructions:**\n"
+                "1. Mimic your own coding style, naming conventions, indentation, and typical error patterns.\n"
+                "2. Introduce mistake you are likely to make (e.g., off‐by‐one index, wrong initialization, "
+                "missing edge case).\n"
+                "3. Do **not** produce a fully correct solution or add unfamiliar optimizations.\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+
+            print(f"\n=== DEBUG INFO FOR STUDENT {student_id}, QUESTION {question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Original pass field: {target.get('pass', 'MISSING')}")
+            print(f"Question template exists: {'question_template' in target}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            # Also add this validation in your UnitTestAlignmentMetric evaluate_generation method:
+            def evaluate_generation(self, adapter_spec, request_state, metric_service, eval_cache_path):
+                print("\n=== UNIT TEST METRIC DEBUG ===")
+                print(f"Has extra_data: {hasattr(request_state.instance, 'extra_data')}")
+                if hasattr(request_state.instance, "extra_data"):
+                    extra_data = request_state.instance.extra_data
+                    print(f"Extra data keys: {list(extra_data.keys())}")
+                    print(f"Test cases: {len(extra_data.get('test_cases', []))}")
+                    print(f"Student pattern: {extra_data.get('student_correctness_pattern', 'MISSING')}")
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response_mistake"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/exams_multilingual_scenario.py

@@ -0,0 +1,115 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    VALID_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class EXAMSMultilingualScenario(Scenario):
+    """EXAMS: A Multi-subject High School Examinations Dataset
+
+    EXAMS is a benchmark dataset for multilingual and cross-lingual
+    question answering from high school examinations. It consists of
+    more than 24,000 high-quality high school exam questions in 16
+    languages, covering 8 language families and 24 school subjects
+    from Natural Sciences and Social Sciences, among others.
+
+    - https://huggingface.co/datasets/mhardalov/exams
+    - https://aclanthology.org/2020.emnlp-main.438/
+
+    Note: Some dataset rows have the value '@' in the `answerKey` column.
+    These rows will be ignored.
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "exams_multilingual"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D", "E"]
+    HF_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "test": TEST_SPLIT, "validation": VALID_SPLIT}
+
+    def __init__(self, language: str, subject: str):
+        super().__init__()
+        self.language = language
+        self.subject = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "mhardalov/exams",
+            "multilingual",
+            revision="4ff10804abb3341f8815cacd778181177bba7edd",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row in dataset:
+                question = row["question"]
+                question_info = row["info"]
+                if self.subject != "all" and question_info["subject"] != self.subject:
+                    continue
+                if self.language != "all" and question_info["language"] != self.language:
+                    continue
+                input = Input(text=question["stem"])
+                references: List[Reference] = []
+                if row["answerKey"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answerKey column in row: {row}")
+                    continue
+                correct_choice_index = ord(row["answerKey"]) - ord("A")
+                for choice_index, choice_text in enumerate(question["choices"]["text"]):
+                    references.append(
+                        Reference(
+                            output=Output(text=choice_text),
+                            tags=[CORRECT_TAG] if choice_index == correct_choice_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances