eval-framework 0.2.7__py3-none-any.whl

eval_framework/tasks/benchmarks/aidanbench.py

@@ -0,0 +1,211 @@
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Union
+
+from eval_framework.metrics.completion.aidanbench import AidanBenchMetric
+from eval_framework.metrics.llm.graders.coherence_grader import CoherenceGrader
+from eval_framework.metrics.llm.graders.language import Language as LLMLanguage
+from eval_framework.shared.types import Completion
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, ResponseType, Sample
+from eval_framework.tasks.base import Language as TaskLanguage
+from eval_framework.utils.helpers import pairwise_cosine_similarity
+from template_formatting.formatter import Message, Role
+
+if TYPE_CHECKING:
+    from eval_framework.llm.base import BaseLLM
+    from eval_framework.shared.types import Error
+
+
+COHERENCE_THRESHOLD = 15
+NOVELTY_THRESHOLD = 0.15
+
+
+class AidanBenchOriginal(BaseTask[str]):
+    """AidanBench (https://openreview.net/pdf?id=fz969ahcvJ)."""
+
+    NAME = "AidanBench"
+    DATASET_PATH = "Aleph-Alpha-Research/aidanbench"
+    SAMPLE_SPLIT = "train"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [AidanBenchMetric]
+    SUBJECTS = [NO_SUBJECT]
+    LANGUAGE = {NO_SUBJECT: TaskLanguage.ENG}
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        from eval_framework.llm.openai import OpenAIEmbeddingModel, OpenAIModel
+
+        super().__init__(num_fewshot)
+        assert num_fewshot == 0, "AidanBench does not support few-shot prompting."
+        self._coherence_grader = CoherenceGrader(grading_model=OpenAIModel(model_name="gpt-4o-mini"))
+        self._embedding_model = OpenAIEmbeddingModel()
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        item_prompt = item["prompt"]
+        # note the extra dot after colon. We take this from the original AidanBench code:
+        # https://github.com/aidanmclaughlin/AidanBench/blob/a6bb3253ff630c82e7adbc81ce7bc7184c5bd881/benchmark/prompts.py#L7  # noqa: E501
+        base_prompt = (
+            "Answer the following question:.\n"
+            "<question>" + item_prompt + "</question>\n"
+            "Provide your answer in <answer></answer> XML tags.\n"
+        )
+        base_prompt += (
+            "Your response should be one direct answer. "
+            "Only provide one answer. DO NOT list multiple answers. Please try to be concise.\n"
+        )
+        return base_prompt
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return None
+
+    def _calculate_novelty_score(self, messages: list[Message]) -> float:
+        assert messages[0].role == Role.USER
+        assert all(msg.role != Role.USER for msg in messages[1:]), "Only the first message should be from USER"
+        messages_without_instruction_ = messages[1:]
+        messages_without_instruction: list[Sequence[Message]] = [
+            [m] for m in messages_without_instruction_
+        ]  # input format for embedding model
+        if len(messages_without_instruction) == 1:
+            return 1.0  # if there's only one response, it's by definition novel
+        all_embeddings = self._embedding_model.generate_embeddings(messages_without_instruction)
+        new_embedding = all_embeddings[-1]
+        previous_embeddings = all_embeddings[:-1]
+        similarities = pairwise_cosine_similarity([new_embedding], previous_embeddings)
+        assert len(similarities) == 1
+        similarities_squeezed = similarities[0]  # "squeeze"
+        assert len(similarities_squeezed) == len(previous_embeddings)
+        return 1 - max(similarities_squeezed)
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return []
+
+    def _fuse_messages(self, messages: list[Message]) -> list[Message]:
+        """
+        Takes a list of messages and fuses them into a single message:
+        A USER message that also contains all previous model responses, wrapped for the next iterative generation step.
+        """
+        assert len(messages) >= 2, "There must be at least one USER and one ASSISTANT message"
+        assert messages[0].role == Role.USER
+        assert all(msg.role == Role.ASSISTANT for msg in messages[1:]), "Only the first message should be from USER"
+
+        instruction_message = messages[0].content
+        previous_answers = [msg.content for msg in messages[1:]]
+
+        previous_answers_str = "\n\n".join(
+            [
+                f"<previous_answer id='{i + 1}'>\n{answer}\n</previous_answer>"
+                for i, answer in enumerate(previous_answers)
+            ]
+        )
+        instruction_message += (
+            "IMPORTANT: Provide an answer you *HAVE NOT* given previously.\n"
+            "Your previous answers are inside of <previous_answers></previous_answers> XML tags.\n"
+            "<previous_answers>\n" + previous_answers_str + "\n</previous_answers>"
+        )
+        return [Message(role=Role.USER, content=instruction_message)]
+
+    def _generation_loop(
+        self, llm: "BaseLLM", stop_sequences: list[str] | None, max_tokens: int | None, initial_samples: list[Sample]
+    ) -> tuple[list[list[Message]], list[Union["Error", None]]]:
+        initial_messages = [s.messages for s in initial_samples]
+        samples = [(s, False) for s in initial_samples]  # (sample, is_done)
+        message_history = [msg for msg in initial_messages]  # to keep track of all iterative model responses
+        errors: list[Error | None] = [None for _ in message_history]
+        while not all(is_done for _, is_done in samples):
+            # iterative generation loop
+            not_done_idx = [i for i, (_, is_done) in enumerate(samples) if not is_done]
+            new_completions = super().generate_completions(
+                llm,
+                [samples[i][0] for i in not_done_idx],
+                stop_sequences=stop_sequences,
+                max_tokens=max_tokens,
+            )
+            new_completion_messages: list[list[Message] | None] = [c.messages for c in new_completions]
+            new_errors = [c.error for c in new_completions]
+
+            new_samples = [s for s in samples]
+            for idx, completion_msgs, error in zip(not_done_idx, new_completion_messages, new_errors):
+                old_sample = samples[idx][0]
+                if completion_msgs is not None:
+                    message_history[idx].append(completion_msgs[-1])  # add latest model response to history
+                    errors[idx] = error
+
+                    assert completion_msgs[0].role == Role.USER and completion_msgs[-1].role == Role.ASSISTANT
+                    coherence_score = self._coherence_grader.grade(
+                        instruction=old_sample.messages[0].content,  # only pass initial instruction
+                        completion=completion_msgs[-1].content,
+                        language=LLMLanguage(iso_639_1="en"),
+                    ).coherence_score
+                else:
+                    coherence_score = 0  # if no completion, treat as non-coherent
+
+                novelty_score = self._calculate_novelty_score(message_history[idx])
+
+                fused_message = self._fuse_messages(message_history[idx])
+                new_sample = Sample(
+                    id=old_sample.id,
+                    subject=old_sample.subject,
+                    ground_truth=old_sample.ground_truth,
+                    messages=fused_message,
+                    context=old_sample.context,
+                    possible_completions=old_sample.possible_completions,
+                )
+                if coherence_score < COHERENCE_THRESHOLD or novelty_score < NOVELTY_THRESHOLD:
+                    # Fail! Stop generating
+                    new_samples[idx] = (new_sample, True)
+                else:
+                    # Continue generating
+                    new_samples[idx] = (new_sample, False)
+            samples = new_samples
+        return message_history, errors
+
+    def generate_completions(
+        self,
+        llm: "BaseLLM",
+        samples: list[Sample],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+    ) -> list[Completion]:
+        assert all(len(s.messages) == 1 and s.messages[0].role == Role.USER for s in samples), (
+            "Each sample must have exactly one USER message."
+        )
+        all_message_histories, errors = self._generation_loop(llm, stop_sequences, max_tokens, samples)
+
+        completion_list = []
+        for idx, sample in enumerate(samples):
+            messages = all_message_histories[idx]
+            error = errors[idx]
+
+            completion_list.append(
+                Completion(
+                    id=sample.id,
+                    subject=sample.subject,
+                    ground_truth=sample.ground_truth,
+                    prompt=sample.messages[0].content,
+                    prompt_sequence_positions=None,
+                    concat_compression=None,
+                    messages=messages,
+                    completion="".join([msg.content for msg in messages if msg.role == Role.ASSISTANT]),
+                    raw_completion="".join([msg.content for msg in messages if msg.role == Role.ASSISTANT]),
+                    raw_completion_sequence_positions=None,
+                    context=sample.context,
+                    error=error,
+                )
+            )
+
+        return completion_list
+
+
+class AidanBench(AidanBenchOriginal):
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        item_prompt = item["prompt"]
+        # We correct the prompt here by removing the extra dot after the colon.
+        base_prompt = (
+            "Answer the following question:\n"
+            "<question>" + item_prompt + "</question>\n"
+            "Provide your answer in <answer></answer> XML tags.\n"
+        )
+        base_prompt += (
+            "Your response should be one direct answer. "
+            "Only provide one answer. DO NOT list multiple answers. Please try to be concise.\n"
+        )
+        return base_prompt

eval_framework/tasks/benchmarks/arc.py

@@ -0,0 +1,70 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import ConfidenceWeightedAccuracy
+from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
+from eval_framework.metrics.loglikelihood.ternary import TernaryScore
+from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.utils import get_n_letters
+
+
+class ARC(BaseTask[str]):
+    """ARC dataset: https://huggingface.co/datasets/allenai/ai2_arc"""
+
+    NAME = "ARC"
+    DATASET_PATH = "ai2_arc"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = ["ARC-Easy", "ARC-Challenge"]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question"] + get_n_letters(5)
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        self.keys = get_n_letters(5)  # needs to be 5 because there is one sample with 5 answer possibilities
+        self.num_to_letter = {str(i): letter for i, letter in enumerate(self.keys, start=1)}
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"Question: {item['question']}\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        answer_key = self.num_to_letter.get(item["answerKey"], item["answerKey"])
+        return f" {item['choices']['text'][self.keys.index(answer_key)]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {choice}" for choice in item["choices"]["text"]]
+
+
+class ARC_IDK(ARC):
+    NAME = "ARC_IDK"
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        ConfidenceWeightedAccuracy,
+        DistributionalCorrectnessScore,
+        TernaryScore,
+    ]
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return (
+            "Answer only if you are confident, since mistakes may be penalised, while correct answers receive points. "
+            "It is acceptable to answer with 'I do not know' if you are unsure, and you will receive 0 points."
+        )
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        completions = super()._get_possible_completions(item)
+        return (completions or []) + [" I do not know."]

eval_framework/tasks/benchmarks/arc_de.py

@@ -0,0 +1,46 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.utils import get_n_letters
+
+
+class ARC_DE(BaseTask[str]):
+    """ARC-DE dataset: https://huggingface.co/datasets/LeoLM/ArcChallenge_de"""
+
+    NAME = "ARC German"
+    DATASET_PATH = "LeoLM/ArcChallenge_de"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "validation"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [NO_SUBJECT]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Frage"] + get_n_letters(5)
+    LANGUAGE = Language.DEU
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        self.keys = get_n_letters(5)  # needs to be 5 because there is one sample with 5 answer possibilities
+        self.num_to_letter = {str(i): letter for i, letter in enumerate(self.keys, start=1)}
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"Frage: {item['question_de']}\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Antwort:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        answer_key = self.num_to_letter.get(item["answerKey"], item["answerKey"])
+        return f" {item['choices_de']['text'][self.keys.index(answer_key)]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {choice}" for choice in item["choices_de"]["text"]]

eval_framework/tasks/benchmarks/arc_fi.py

@@ -0,0 +1,46 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.utils import get_n_letters
+
+
+class ARC_FI(BaseTask[str]):
+    """ARC-FI dataset: https://huggingface.co/datasets/LumiOpen/arc_challenge_mt"""
+
+    NAME = "ARC Finnish"
+    DATASET_PATH = "LumiOpen/arc_challenge_mt"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "validation"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = ["fi"]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question"] + get_n_letters(5)
+    LANGUAGE = Language.FIN
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        self.keys = get_n_letters(5)  # needs to be 5 because there is one sample with 5 answer possibilities
+        self.num_to_letter = {str(i): letter for i, letter in enumerate(self.keys, start=1)}
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return f"Question: {item['question']}\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        answer_key = self.num_to_letter.get(item["answerKey"], item["answerKey"])
+        return f" {item['choices']['text'][self.keys.index(answer_key)]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {choice}" for choice in item["choices"]["text"]]

eval_framework/tasks/benchmarks/belebele.py

@@ -0,0 +1,60 @@
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.utils import get_n_letters
+
+
+class BELEBELE(BaseTask[str]):
+    """BELEBELE dataset: https://huggingface.co/datasets/facebook/belebele"""
+
+    NAME = "BELEBELE"
+    DATASET_PATH = "facebook/belebele"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = [
+        "eng_Latn",
+    ]
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"] + get_n_letters(4)
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        super().__init__(num_fewshot)
+
+        self.keys = get_n_letters(4)
+        self.num_to_letter = {str(i): letter for i, letter in enumerate(self.keys, start=1)}
+
+    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
+        return "The following are multiple choice questions (with answers)."
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        context = item["flores_passage"].strip()
+        question = item["question"].strip()
+        choices = "".join(
+            [
+                f"{key}. {choice}\n"
+                for key, choice in zip(
+                    self.keys, [item["mc_answer1"], item["mc_answer2"], item["mc_answer3"], item["mc_answer4"]]
+                )
+            ]
+        )
+        return f"{context}\n\nQuestion: {question}\n{choices}"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        ground_truth = self._get_ground_truth(item)
+        assert ground_truth is not None
+        return f"{self._get_cue_text(item)}{ground_truth}"
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return f" {self.keys[int(item['correct_answer_num']) - 1]}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return [f" {key}" for key in self.keys]

eval_framework/tasks/benchmarks/bigcodebench.py

@@ -0,0 +1,155 @@
+import random
+import re
+from typing import Any
+
+from eval_framework.metrics.completion.code_execution_pass_at_one import (
+    CodeExecutionPassAtOne,
+    CodeExecutionPassAtOneContext,
+)
+from eval_framework.tasks.base import (
+    RANDOM_SEED,
+    BaseTask,
+    Language,
+    ResponseType,
+    Sample,
+    SubjectType,
+)
+from eval_framework.tasks.utils import (
+    BIG_CODE_BENCH_PACKAGE_MAPPING,
+    CallableSerializer,
+    _parse_unittest_output,
+    unittest_merge_snippets,
+)
+
+PROMPT_INSTRUCTION = (
+    "Please provide a self-contained Python script, without tests or example usage, that solves the following "
+    "problem in a markdown code block:\n"
+)  # from https://arxiv.org/pdf/2406.15877 - Figure 14
+
+
+RESPONSE_PREFIX = (
+    "Below is a Python script with a self-contained function that solves the problem and passes "
+    "corresponding tests:\n"
+)  # from https://github.com/bigcode-project/bigcodebench/blob/main/bigcodebench/generate.py#L149
+
+
+class BigCodeBench(BaseTask[str]):
+    """BigCodeBench dataset: https://huggingface.co/datasets/bigcode/bigcodebench"""
+
+    NAME = "BigCodeBench"
+    DATASET_PATH = "bigcode/bigcodebench"
+    SAMPLE_SPLIT = "v0.1.4"
+    FEWSHOT_SPLIT = "v0.1.4"  # (there is no dedicated split, few-shot is not expected for this dataset)
+    RESPONSE_TYPE = ResponseType.COMPLETION
+    METRICS = [CodeExecutionPassAtOne]
+    SUBJECTS = ["original", "calibrated"]
+    LANGUAGE = Language.ENG
+
+    def __init__(self, num_fewshot: int = 0) -> None:
+        assert num_fewshot == 0, "Fewshot is not supported for BigCodeBench"
+        # NOTE : this serializer should be the same class as initialized in the metric
+        self.serializer = CallableSerializer()
+        super().__init__(num_fewshot)
+
+    def _load_dataset(self, subject: SubjectType) -> None:
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name=None)
+        self.dataset = {}
+
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            data_list = list(data)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            if split in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                self.dataset[split] = data_list
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return PROMPT_INSTRUCTION + item["complete_prompt"]
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return RESPONSE_PREFIX + (item["code_prompt"] if item["subject"] == "calibrated" else "")
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return item["canonical_solution"]  # Not needed for evaluation, as it is test based given the generated code
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        return None
+
+    def _get_context(self, item: dict[str, Any]) -> CodeExecutionPassAtOneContext:
+        return CodeExecutionPassAtOneContext(
+            run_env="python:3.12",  # os.environ.get("DOCKER_CODE_EXECUTION"),
+            code_prompt=item["code_prompt"],
+            test_code=item["test"],
+            snippet_merge_fn=self.serializer.encode(unittest_merge_snippets),
+            output_parse_fn=self.serializer.encode(_parse_unittest_output),
+            package_downloads=BIG_CODE_BENCH_PACKAGE_MAPPING,
+        )
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        if sample is not None and sample.context is not None and sample.subject == "calibrated":
+            assert isinstance(sample.context, CodeExecutionPassAtOneContext), "Expected CodeExecutionPassAtOneContext"
+            processed_text = (sample.context.code_prompt if sample.context is not None else "") + completion_text
+        else:
+            processed_text = extract_executable_code(completion_text)
+
+        return processed_text
+
+
+class BigCodeBenchInstruct(BigCodeBench):
+    """BigCodeBench dataset: https://huggingface.co/datasets/bigcode/bigcodebench"""
+
+    NAME = "BigCodeBenchInstruct"
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return PROMPT_INSTRUCTION + item["instruct_prompt"]
+
+
+class BigCodeBenchHard(BigCodeBench):
+    """BigCodeBench dataset: https://huggingface.co/datasets/bigcode/bigcodebench-hard"""
+
+    NAME = "BigCodeBenchHard"
+    DATASET_PATH = "bigcode/bigcodebench-hard"
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return PROMPT_INSTRUCTION + item["complete_prompt"]
+
+
+class BigCodeBenchHardInstruct(BigCodeBenchHard):
+    """BigCodeBench dataset: https://huggingface.co/datasets/bigcode/bigcodebench-hard"""
+
+    NAME = "BigCodeBenchHardInstruct"
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return PROMPT_INSTRUCTION + item["instruct_prompt"]
+
+
+def extract_executable_code(llm_response: str) -> str:
+    # Look for nested markdown+python pattern
+    nested_pattern = r"```markdown.*?```python\s*(.*?)\s*```"
+    nested_matches = re.findall(nested_pattern, llm_response, re.DOTALL)
+    if nested_matches:
+        return nested_matches[0].strip()
+
+    # Look for python code blocks
+    python_pattern = r"```python\s*(.*?)\s*```"
+    python_matches = re.findall(python_pattern, llm_response, re.DOTALL)
+    if python_matches:
+        return python_matches[0].strip()
+
+    # Look for markdown-only code blocks
+    markdown_pattern = r"```markdown\s*(.*?)\s*```"
+    markdown_matches = re.findall(markdown_pattern, llm_response, re.DOTALL)
+    if markdown_matches:
+        return markdown_matches[0].strip()
+
+    # Look for generic code blocks as fallback
+    generic_pattern = r"```\s*(.*?)\s*```"
+    generic_matches = re.findall(generic_pattern, llm_response, re.DOTALL)
+    if generic_matches:
+        return generic_matches[0].strip()
+
+    # If no code blocks found, return original response
+    return llm_response

eval_framework/tasks/benchmarks/casehold.py

@@ -0,0 +1,47 @@
+import random
+from typing import Any
+
+from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyLoglikelihood,
+    AccuracyNormLoglikelihood,
+)
+from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType
+
+
+class CASEHOLD(BaseTask[str]):
+    NAME = "CaseHold"
+    DATASET_PATH = "lex_glue"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "train"
+    RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    SUBJECTS = ["case_hold"]
+    LANGUAGE = Language.ENG
+
+    def _load_dataset(self, subject: str) -> None:
+        name = subject if subject != NO_SUBJECT else None
+
+        hf_dataset = self._load_hf_dataset(path=self.DATASET_PATH, name=name)
+        self.dataset = {}
+
+        self.rnd = random.Random(RANDOM_SEED)
+
+        for split, data in hf_dataset.items():
+            data_list = list(data)
+
+            if split == self.SAMPLE_SPLIT:
+                self.rnd.shuffle(data_list)
+
+            if split in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
+                self.dataset[split] = [i for i in data_list if i["context"].count("(<HOLDING>)") == 1]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return item["context"].split("(<HOLDING>)", maxsplit=1)[0]
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        right = item["context"].split("(<HOLDING>)", maxsplit=1)[1]
+        return f"{item['endings'][item['label']]}{right}"
+
+    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
+        right = item["context"].split("(<HOLDING>)", maxsplit=1)[1]
+        return [f"{ending}{right}" for ending in item["endings"]]