eval-framework 0.2.13__tar.gz → 0.2.14__tar.gz

{eval_framework-0.2.13 → eval_framework-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.2.13
+Version: 0.2.14
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.2.13 → eval_framework-0.2.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.2.13"
+version = "0.2.14"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/llm/aleph_alpha.py

@@ -200,11 +200,16 @@ class AlephAlphaAPIModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         effective_temperature = temperature if temperature is not None else self._temperature
+        if effective_temperature is not None and not (0 <= effective_temperature <= 2):
+            raise ValueError(f"temperature must be between 0 and 2, got {effective_temperature}")
+        effective_top_p = top_p if top_p is not None else self._top_p
+        if effective_top_p is not None and not (0 < effective_top_p <= 1):
+            raise ValueError(f"top_p must be between 0 and 1 (exclusive), got {effective_top_p}")
 
         requests: list[CompletionRequest] = []
-
         # Adjust max tokens based on bytes_per_token_scalar so that non-standard models generate full responses
         scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
 
@@ -215,7 +220,7 @@ class AlephAlphaAPIModel(BaseLLM):
                     maximum_tokens=scaled_max_tokens,
                     stop_sequences=stop_sequences,
                     temperature=effective_temperature,
-                    top_p=self._top_p,
+                    top_p=effective_top_p,
                 )
             )
 

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/llm/base.py

@@ -24,6 +24,7 @@ class BaseLLM(ABC):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """
         stop_sequences and max_tokens are injected by the task if exist. They should be overwritten or
@@ -47,6 +48,7 @@ class BaseLLM(ABC):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """
         stop_sequences and max_tokens are injected by the task if exist. They should be overwritten or
@@ -79,6 +81,7 @@ class BaseLLM(ABC):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """Generates a model response for each sample.
 
@@ -86,10 +89,10 @@ class BaseLLM(ABC):
         otherwise falls back to 'generate_from_messages'.
         """
         try:
-            return self.generate_from_samples(samples, stop_sequences, max_tokens, temperature)
+            return self.generate_from_samples(samples, stop_sequences, max_tokens, temperature, top_p)
         except NotImplementedError:
             messages: list[Sequence[Message]] = [sample.messages for sample in samples]
-            return self.generate_from_messages(messages, stop_sequences, max_tokens, temperature)
+            return self.generate_from_messages(messages, stop_sequences, max_tokens, temperature, top_p)
 
     def post_process_completion(self, completion: str, sample: Sample) -> str:
         """

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/llm/huggingface.py

@@ -146,6 +146,7 @@ class BaseHFLLM(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         if temperature is None:
             effective_temperature = 0.0  # Current default, TODO: refactor to use model's default
@@ -154,7 +155,8 @@ class BaseHFLLM(BaseLLM):
             )
         else:
             effective_temperature = temperature
-
+        if top_p is not None:
+            logger.warning("Huggingface LLM does not support top_p. Ignoring top_p value.")
         raw_completions = []
         for single_messages in messages:
             # format

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/llm/openai.py

@@ -34,6 +34,7 @@ class OpenAIModel(BaseLLM):
         model_name: str | None = None,
         formatter: BaseFormatter | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
         api_key: str | None = os.getenv("OPENAI_API_KEY", ""),
         organization: str | None = None,
         base_url: str | None = None,
@@ -46,6 +47,7 @@ class OpenAIModel(BaseLLM):
             model_name: OpenAI model name (e.g., "gpt-4o", "gpt-3.5-turbo"). If None, uses LLM_NAME class attribute.
             formatter: Optional message formatter.
             temperature: Sampling temperature used when not passed to generate methods (from 0.0 to 2.0).
+            top_p: Nucleus sampling probability mass (from 0.0 to 1.0). If None, the API default is used.
             api_key: OpenAI API key (defaults to OPENAI_API_KEY env variable).
             organization: Optional OpenAI organization ID.
             base_url: Optional API base URL for Azure or alternate endpoints.
@@ -59,6 +61,10 @@ class OpenAIModel(BaseLLM):
         self._temperature = temperature if temperature is not None else 0.0
         assert 0.0 <= self._temperature <= 2.0, "Temperature must be between 0.0 and 2.0"
 
+        if top_p is not None:
+            assert 0.0 <= top_p <= 1.0, "top_p must be between 0.0 and 1.0"
+        self._top_p = top_p
+
         self._client = OpenAI(
             api_key=api_key,
             organization=organization,
@@ -97,6 +103,7 @@ class OpenAIModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         """
         Generate completions for a list of message sequences concurrently.
@@ -108,6 +115,7 @@ class OpenAIModel(BaseLLM):
             stop_sequences: Optional list of stop sequences.
             max_tokens: Optional maximum number of tokens to generate.
             temperature: Sampling temperature.
+            top_p: Nucleus sampling probability mass (0.0 to 1.0). Overrides instance default if provided.
 
         Returns:
             List of RawCompletion objects containing prompts and completions.
@@ -116,6 +124,10 @@ class OpenAIModel(BaseLLM):
         effective_temperature = temperature if temperature is not None else self._temperature
         assert 0.0 <= effective_temperature <= 2.0, "Temperature must be between 0.0 and 2.0"
 
+        effective_top_p = top_p if top_p is not None else self._top_p
+        if effective_top_p is not None:
+            assert 0.0 <= effective_top_p <= 1.0, "top_p must be between 0.0 and 1.0"
+
         def _process_one(single_messages: Sequence[Message]) -> RawCompletion:
             # Adjust max tokens based on bytes_per_token_scalar so that non-standard models generate full responses
             scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
@@ -129,6 +141,7 @@ class OpenAIModel(BaseLLM):
                     model=self._model_name,
                     prompt=prompt,
                     temperature=effective_temperature,
+                    top_p=effective_top_p,
                     max_tokens=scaled_max_tokens,
                     stop=stop_sequences,
                 )
@@ -158,6 +171,7 @@ class OpenAIModel(BaseLLM):
                     model=self._model_name,
                     messages=chat_messages,
                     temperature=effective_temperature,
+                    top_p=effective_top_p,
                     max_tokens=scaled_max_tokens,
                     stop=stop_sequences,
                 )
@@ -300,6 +314,7 @@ class OpenAIEmbeddingModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         raise NotImplementedError(
             "Embedding model does not support generate_from_messages. Use generate_embeddings instead."

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/llm/vllm.py

@@ -226,6 +226,7 @@ class BaseVLLMModel(BaseLLM):
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        top_p: float | None = None,
     ) -> list[RawCompletion]:
         raw_completions: list[RawCompletion | None] = [None] * len(messages)
         prompt_objs = []
@@ -235,7 +236,7 @@ class BaseVLLMModel(BaseLLM):
         scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
 
         sampling_params = self._resolve_sampling_params(
-            self.sampling_params, scaled_max_tokens, stop_sequences, temperature
+            self.sampling_params, scaled_max_tokens, stop_sequences, temperature, top_p
         )
 
         for i, single_messages in enumerate(messages):
@@ -295,6 +296,7 @@ class BaseVLLMModel(BaseLLM):
         max_tokens: int | None,
         stop_sequences: list[str] | None,
         temperature: float | None,
+        top_p: float | None = None,
     ) -> SamplingParams:
         sampling_params.max_tokens = max_tokens
         sampling_params.stop = stop_sequences
@@ -308,6 +310,13 @@ class BaseVLLMModel(BaseLLM):
                 f"Using sampling params temperature value: {sampling_params.temperature} "
                 f"as no custom temperature value was provided"
             )
+        if top_p is not None:
+            logger.warning(f"Overriding sampling params top_p {sampling_params.top_p} with custom value {top_p}")
+            sampling_params.top_p = top_p
+        else:
+            logger.info(
+                f"Using sampling params top_p value: {sampling_params.top_p} as no custom top_p value was provided"
+            )
         return sampling_params
 
     def _model_generate(

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -394,6 +394,25 @@ class AIME2025(AIME2024):
         return item["answer"]
 
 
+class AIME2026(AIME2024):
+    """AIME 2026 dataset: https://huggingface.co/datasets/math-ai/aime26
+
+    This dataset contains a single test split of 30 questions.
+    Data contains
+    problem | answer | id
+
+    pass@1 evaluation
+    """
+
+    NAME = "AIME2026"
+    DATASET_PATH = "math-ai/aime26"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
+        return item["answer"]
+
+
 class MATH500(MATHReasoning):
     """MATH500 dataset: https://huggingface.co/datasets/HuggingFaceH4/MATH-500
 

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -105,9 +105,7 @@ class MBPP(BaseTask[str]):
     def _get_context(self, item: dict[str, Any]) -> MBPPMetricContext:
         return MBPPMetricContext(tests_code="\n".join(item["test_list"]))
 
-    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
-        assert sample is not None
-
+    def post_process_generated_completion(self, completion_text: str, sample: Sample) -> str:  # type: ignore[override]
         if BEGIN in completion_text:
             completion_text = completion_text.split(f"{BEGIN}\n")[1]
 
@@ -193,9 +191,7 @@ class MBPP_PROMPT_WITHOUT_TESTS(MBPP):
             return postfix
         return f"{postfix.strip()}:"
 
-    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
-        assert sample is not None
-
+    def post_process_generated_completion(self, completion_text: str, sample: Sample) -> str:  # type: ignore[override]
         if BEGIN in completion_text:
             completion_text = completion_text.split(BEGIN)[1]
 
@@ -212,3 +208,102 @@ class MBPP_PROMPT_WITHOUT_TESTS(MBPP):
 class MBPP_PROMPT_WITHOUT_TESTS_SANITIZED(MBPP_PROMPT_WITHOUT_TESTS):
     NAME = "MBPP_PROMPT_WITHOUT_TESTS_SANITIZED"
     SUBJECTS = ["sanitized"]
+
+
+_OLMES_FEWSHOT_EXAMPLES: list[dict[str, Any]] = [
+    {
+        "text": "Write a function to find the similar elements from the given two tuple lists.",
+        "code": (
+            "def similar_elements(test_tup1, test_tup2):\n"
+            "  res = tuple(set(test_tup1) & set(test_tup2))\n  return (res)"
+        ),
+        "test_list": [
+            "assert similar_elements((3, 4, 5, 6),(5, 7, 4, 10)) == (4, 5)",
+            "assert similar_elements((1, 2, 3, 4),(5, 4, 3, 7)) == (3, 4)",
+            "assert similar_elements((11, 12, 14, 13),(17, 15, 14, 13)) == (13, 14)",
+        ],
+    },
+    {
+        "text": "Write a python function to identify non-prime numbers.",
+        "code": (
+            "import math\ndef is_not_prime(n):\n    result = False\n"
+            "    for i in range(2,int(math.sqrt(n)) + 1):\n"
+            "        if n % i == 0:\n            result = True\n    return result"
+        ),
+        "test_list": [
+            "assert is_not_prime(2) == False",
+            "assert is_not_prime(10) == True",
+            "assert is_not_prime(35) == True",
+        ],
+    },
+    {
+        "text": (
+            "Write a function to find the largest integers from a given list of numbers using heap queue algorithm."
+        ),
+        "code": (
+            "import heapq as hq\ndef heap_queue_largest(nums,n):\n"
+            "  largest_nums = hq.nlargest(n, nums)\n  return largest_nums"
+        ),
+        "test_list": [
+            "assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65] ",
+            "assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],2)==[85, 75] ",
+            "assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],5)==[85, 75, 65, 58, 35]",
+        ],
+    },
+]
+
+
+class MBPP_OLMES(MBPP):
+    """
+    MBPP OLMES variant replicating oe_eval's ``mbpp:3shot::olmo3:n32:v2``.
+
+    Uses the EvalPlus prompt format with 3 hardcoded fewshot examples from the
+    original MBPP "prompt" split (matching oe_eval's ordering). Each prompt
+    shows one test case (the first) instead of all.
+
+    Recommended EvalConfig settings for full replication::
+
+        split: test
+        num_fewshot: 3 (hardcoded, prompt split)
+        metric: pass_at_1
+        temperature: 0.6
+        top_p: 0.6
+        repeats: 32
+    """
+
+    NAME = "MBPP_OLMES"
+    FEWSHOT_SPLIT = "test"
+
+    def __init__(self, num_fewshot: int = 3) -> None:
+        super().__init__(num_fewshot)
+        assert num_fewshot == 3, "MBPP_OLMES requires exactly 3 fewshot examples"
+        self.stop_sequences = ["```", '\n"""', "\nassert", "\n#"]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        text = item["text"] if "text" in item else item["prompt"]
+        test = item["test_list"][0]
+        return (
+            "Please provide a self-contained Python script that solves the following problem"
+            f" in a markdown code block:\n```\n{text.strip()}\n{test}\n```\n"
+        )
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Here is the completed function:\n\n```python\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return item["code"] + "\n"
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return list(_OLMES_FEWSHOT_EXAMPLES)
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample) -> str:  # type: ignore[override]
+        assert self.stop_sequences is not None
+
+        for stop_seq in self.stop_sequences:
+            if stop_seq in completion_text:
+                completion_text = completion_text.split(stop_seq)[0]
+
+        extracted_code = completion_text + "\n"
+        mbpp_ground_truth = str(sample.ground_truth)
+        code = self._code_expander(extracted_code, mbpp_ground_truth)
+        return code

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/tasks/task_names.py

@@ -19,6 +19,7 @@ def register_all_tasks() -> None:
     """Register all the benchmark tasks with the eval framework."""
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025")
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2026")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_OLMES")
@@ -88,6 +89,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_SANITIZED")
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_PROMPT_WITHOUT_TESTS")
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_PROMPT_WITHOUT_TESTS_SANITIZED")
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES")

{eval_framework-0.2.13 → eval_framework-0.2.14}/src/eval_framework/utils/generate_task_docs.py

@@ -74,13 +74,17 @@ def generate_docs_for_task(
     try:
         num_fewshot = 1
         task = task_class(num_fewshot=num_fewshot)
-    except Exception:
+    except (TypeError, ValueError, AssertionError):
         try:
             num_fewshot = 0
             task = task_class(num_fewshot=num_fewshot)
-        except Exception as e:
-            print(f"Failed to instantiate task {task_name}: {e}")
-            return
+        except (TypeError, ValueError, AssertionError):
+            try:
+                task = task_class()
+                num_fewshot = task.num_fewshot
+            except Exception as e:
+                print(f"Failed to instantiate task {task_name}: {e}")
+                return
 
     with open(f"{output_docs_directory}/{task_name}.md", "w") as f:
         f.write(f"# {task_name}\n\n")