eval-framework 0.9.1__tar.gz → 0.9.2__tar.gz

{eval_framework-0.9.1 → eval_framework-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.9.1
+Version: 0.9.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -216,7 +216,7 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.1,<4
+Requires-Dist: nltk>=3.10.2,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2

{eval_framework-0.9.1 → eval_framework-0.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.9.1"
+version = "0.9.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -23,7 +23,7 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.1,<4",
+  "nltk>=3.10.2,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/metrics/completion/code_assertion.py

@@ -2,7 +2,7 @@ from llm_sandbox.exceptions import SandboxTimeoutError
 
 from eval_framework.metrics.base import BaseMetric, MetricResult
 from eval_framework.shared.types import Completion
-from eval_framework.tasks.utils import run_python_code
+from eval_framework.tasks.utils import DockerReturnedEmptyOutput, run_python_code
 
 
 class CodeCompletionAssertion(BaseMetric[Completion]):
@@ -16,7 +16,7 @@ class CodeCompletionAssertion(BaseMetric[Completion]):
         code = response.completion
         try:
             output = run_python_code(code, image="python:3.12-slim", runtime_configs={"mem_limit": "512m"})
-        except SandboxTimeoutError:
+        except (SandboxTimeoutError, DockerReturnedEmptyOutput):
             # The submitted code timed out (e.g. an infinite loop) -- a failing sample, not an infra
             # problem.
             import traceback

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/metrics/completion/minerva_math_utils.py

@@ -350,8 +350,17 @@ def strip_string_hendrycks(string: str) -> str:
     return string
 
 
+# MATH gold answers stay under ~80 chars.
+_PLAUSIBLE_ANSWER_LEN = 128
+
+
 def is_equiv_minerva(x1: str, x2: str, timeout_seconds: int = 5) -> bool:
     """Sympy-based equivalence (Minerva)."""
+    # extract_answers can fall back to the whole completion; sympy grinds on such
+    # 1000+ char strings until the timeout. A side far longer than the other can't
+    # be an equivalent final answer, so refuse it before parsing.
+    if max(len(x1), len(x2)) > max(_PLAUSIBLE_ANSWER_LEN, 3 * min(len(x1), len(x2))):
+        return False
 
     def _timeout_handler(signum: Any, frame: Any) -> None:
         raise TimeoutError()

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/response_generator.py

@@ -26,7 +26,7 @@ from eval_framework.shared.types import (
     Loglikelihood,
     RawLoglikelihood,
 )
-from eval_framework.tasks.base import Language, ResponseType, Sample
+from eval_framework.tasks.base import RANDOM_SEED, Language, ResponseType, Sample
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.utils import raise_errors
 from eval_framework.utils.constants import RED, RESET
@@ -67,6 +67,7 @@ class ResponseGenerator:
                 config.task_subjects,
                 config.hf_revision,
                 user_prompt_suffix=config.user_prompt_suffix,
+                seed=RANDOM_SEED,
             )
         else:
             self.task = registry()[config.task_name].create(
@@ -74,6 +75,7 @@ class ResponseGenerator:
                 config.task_subjects,
                 config.hf_revision,
                 user_prompt_suffix=config.user_prompt_suffix,
+                seed=RANDOM_SEED,
             )
 
         self.response_type = self.task.get_response_type()

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/tasks/base.py

@@ -147,6 +147,7 @@ class BaseTask[SubjectType](Task):
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
         self.hf_revision: str | None = self._apply_hf_revision()
+        self.rnd: random.Random | None = None
 
     def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> str | None:
         # Precedence: CLI/config override > REVISION_LOCKFILE pin.
@@ -167,12 +168,13 @@ class BaseTask[SubjectType](Task):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = RANDOM_SEED,
     ) -> Self:
         instance = cls(num_fewshot=num_fewshot)
         if user_prompt_suffix is not None and instance.get_response_type() != ResponseType.COMPLETION:
             raise ValueError("user_prompt_suffix is only supported for completion tasks.")
         instance.user_prompt_suffix = user_prompt_suffix
-
+        instance.rnd = random.Random(seed)
         # If custom subjects were provided during initialization, they take precedence over the class-level SUBJECTS.
         filtered_subjects = instance._filter_task_subjects(custom_subjects=custom_subjects)
         if filtered_subjects:
@@ -244,8 +246,8 @@ class BaseTask[SubjectType](Task):
         )
 
     def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, Any]:
+        assert self.rnd is not None, "Task RNG is unseeded; build tasks via `with_overwrite`."
         dataset = {}
-        self.rnd = random.Random(RANDOM_SEED)
 
         for split, data in hf_dataset.items():
             if split not in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
@@ -413,6 +415,7 @@ class BaseTask[SubjectType](Task):
         return None
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        assert self.rnd is not None, "Task RNG is unseeded; build tasks via `with_overwrite`."
         if self.FEWSHOT_SPLIT == self.SAMPLE_SPLIT:
             # If the fewshot and sample splits are the same, we risk including the current eval item
             # as a fewshot example (leaking the answer). To prevent this, sample one extra example,

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -102,6 +102,7 @@ class MBPP(BaseTask[str]):
         return f"{BEGIN}\n" + target + f"\n{END}"
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        assert self.rnd is not None
         fewshot_examples = self.rnd.sample(self.dataset[self.FEWSHOT_SPLIT], self.num_fewshot)
         return fewshot_examples
 

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/tasks/registry.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Generator, Iterator, Sequence
 from typing import TYPE_CHECKING, Any
 
-from eval_framework.tasks.base import BaseTask, ResponseType
+from eval_framework.tasks.base import RANDOM_SEED, BaseTask, ResponseType
 from eval_framework.tasks.perturbation import PerturbationConfig, create_perturbation_class
 from template_formatting.formatter import BaseFormatter
 
@@ -62,6 +62,7 @@ class EvalFactory(ABC):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask: ...
 
     @abstractmethod
@@ -72,6 +73,7 @@ class EvalFactory(ABC):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask: ...
 
     @abstractmethod
@@ -115,12 +117,14 @@ class _Lazy(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         return self.task_class().with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def create_perturbation(
@@ -130,6 +134,7 @@ class _Lazy(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
         return perturbation_task_class.with_overwrite(
@@ -137,6 +142,7 @@ class _Lazy(EvalFactory):
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def response_type(self) -> ResponseType:
@@ -153,9 +159,9 @@ class _Lazy(EvalFactory):
 
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         return task.markdown_doc(formatters)
 
 
@@ -178,12 +184,14 @@ class _Eager(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         return self._task.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def create_perturbation(
@@ -193,6 +201,7 @@ class _Eager(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         perturbation_task_class = create_perturbation_class(self._task, perturbation_config)
         return perturbation_task_class.with_overwrite(
@@ -200,6 +209,7 @@ class _Eager(EvalFactory):
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def response_type(self) -> ResponseType:
@@ -216,9 +226,9 @@ class _Eager(EvalFactory):
 
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         return task.markdown_doc(formatters)
 
 

{eval_framework-0.9.1 → eval_framework-0.9.2}/src/eval_framework/tasks/utils.py

@@ -58,6 +58,7 @@ def get_or_create_pool(
     lang: str = "python",
     min_pool_size: int = 1,
     max_pool_size: int = 1,
+    max_container_uses: int = 100,
     runtime_configs: dict[str, str] | None = None,
 ) -> ContainerPoolManager:
     assert image or dockerfile, "Either image or dockerfile must be provided"
@@ -65,7 +66,9 @@ def get_or_create_pool(
     with _pools_lock:
         if key not in _pools:
             pool = create_pool_manager(
-                config=PoolConfig(min_pool_size=min_pool_size, max_pool_size=max_pool_size),
+                config=PoolConfig(
+                    min_pool_size=min_pool_size, max_container_uses=max_container_uses, max_pool_size=max_pool_size
+                ),
                 lang=lang,
                 image=image,
                 dockerfile=dockerfile,
@@ -105,6 +108,10 @@ def get_n_letters(n: int) -> list[str]:
     return list(string.ascii_uppercase)[: max(0, n)]
 
 
+class DockerReturnedEmptyOutput(Exception):
+    """Raised when the docker returns an empty output."""
+
+
 def run_python_code(
     code: str,
     image: str | None = None,
@@ -142,6 +149,9 @@ def run_python_code(
         out = (output.stderr + output.stdout).strip()
         if isinstance(out, bytes):
             out = out.decode("utf-8")
+
+        if not out.strip():
+            raise DockerReturnedEmptyOutput("Docker returned an empty output.")
         return out