eval-framework 0.8.0__tar.gz → 0.8.2__tar.gz

{eval_framework-0.8.0 → eval_framework-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.0
+Version: 0.8.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -246,7 +246,7 @@ Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
-Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
+Requires-Dist: torch>=2.13.0,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
 Requires-Python: >=3.12, <3.13
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework
@@ -528,6 +528,8 @@ If you use `eval-framework` in your research, please cite:
 
 This project is licensed under the [Apache License 2.0](LICENSE).
 
+The constituent tasks' datasets can be subject to specific, more restrictive license terms by third parties. By using `eval-framework` you agree to adhere to and be bound by such license terms in the individual case.
+
 <br><br>
 ---
 

{eval_framework-0.8.0 → eval_framework-0.8.2}/README.md

@@ -267,6 +267,8 @@ If you use `eval-framework` in your research, please cite:
 
 This project is licensed under the [Apache License 2.0](LICENSE).
 
+The constituent tasks' datasets can be subject to specific, more restrictive license terms by third parties. By using `eval-framework` you agree to adhere to and be bound by such license terms in the individual case.
+
 <br><br>
 ---
 

{eval_framework-0.8.0 → eval_framework-0.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.0"
+version = "0.8.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -60,7 +60,7 @@ openai = [
 ]
 transformers = [
   "transformers>=4.45.2,<5",
-  "torch>=2.5,<3",
+  "torch>=2.13.0,<3",
   "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]

{eval_framework-0.8.0 → eval_framework-0.8.2}/src/eval_framework/evaluation_generator.py

@@ -20,7 +20,7 @@ from eval_framework.tasks.base import ResponseType
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.registry import registry
 from eval_framework.utils.constants import RED, RESET
-from eval_framework.utils.tqdm_handler import get_disable_bar_flag, safe_tqdm_write
+from eval_framework.utils.tqdm_handler import get_disable_bar_flag
 
 logger = logging.getLogger(__name__)
 
@@ -79,7 +79,6 @@ class EvaluationGenerator:
             metric.fail_on_error = self.config.fail_on_error
 
             logger.info(f"Starting calculation of {metric.NAME}")
-            safe_tqdm_write(f"INFO: Calculating {metric.NAME}")
             for response in tqdm(responses, desc=f"Calculating {metric.NAME}", disable=get_disable_bar_flag()):
                 if f"{response.subject}_{response.id}_{metric.__class__.__name__}" in subject_result_id_existing:
                     continue
@@ -117,7 +116,6 @@ class EvaluationGenerator:
                         self.result_processor.save_metrics_result(result)
 
             logger.info(f"Completed calculation of {metric.NAME}")
-            safe_tqdm_write(f"INFO: Completed {metric.NAME}")
 
         if not self.save_intermediate_results:
             self.result_processor.save_metrics_results(results)

{eval_framework-0.8.0 → eval_framework-0.8.2}/src/eval_framework/response_generator.py

@@ -30,7 +30,7 @@ from eval_framework.tasks.base import Language, ResponseType, Sample
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.utils import raise_errors
 from eval_framework.utils.constants import RED, RESET
-from eval_framework.utils.tqdm_handler import get_disable_bar_flag, safe_tqdm_write
+from eval_framework.utils.tqdm_handler import get_disable_bar_flag
 
 logger = logging.getLogger(__name__)
 
@@ -232,9 +232,7 @@ class ResponseGenerator:
             if not samples_batch:
                 return
             if len(samples_batch) > 1:
-                log_msg = "Processing batch..."
-                logger.info(log_msg)  # For log files
-                safe_tqdm_write(log_msg)  # For console display with tqdm
+                logger.info("Processing batch...")
 
             responses_batch = generative_output_function(samples_batch)
             responses.extend(responses_batch)
@@ -268,17 +266,13 @@ class ResponseGenerator:
                 sample_index = i + 1
 
                 if sample.id in subject_response_id_mapping.get(sample.subject, []):
-                    log_msg = (
+                    logger.info(
                         f"Task: {self.response_type.value}{subject} - Sample: {sample_index} - skipping, already done."
                     )
-                    logger.info(log_msg)  # For log files
-                    safe_tqdm_write(log_msg)  # For console display with tqdm
                     pbar.update(1)
                     continue
 
-                log_msg = f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}"
-                logger.info(log_msg)  # For log files
-                safe_tqdm_write(log_msg)  # For console display with tqdm
+                logger.info(f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}")
                 pbar.set_postfix_str(f"Sample {sample_index}/{total_num_samples}")
                 pbar.update(1)
 
@@ -289,9 +283,7 @@ class ResponseGenerator:
                     samples_batch = []
 
                 if should_preempt_callable():
-                    log_msg = "Preempt"
-                    logger.info(log_msg)  # For log files
-                    safe_tqdm_write(log_msg)  # For console display with tqdm
+                    logger.info("Preempt")
                     if not self.save_intermediate_results:
                         self.result_processor.save_responses(responses)
                     return responses, True

{eval_framework-0.8.0 → eval_framework-0.8.2}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -10,10 +10,7 @@ from eval_framework.metrics.completion.math_minerva_completion import (
     MathMinervaCompletionRelaxed,
 )
 from eval_framework.metrics.completion.math_reasoning_completion import MathReasoningCompletion
-from eval_framework.metrics.completion.minerva_math_utils import (
-    extract_answers,
-    normalized_gold_from_solution,
-)
+from eval_framework.metrics.completion.minerva_math_utils import extract_answers, normalized_gold_from_solution
 from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.task_style import BPBStyle
@@ -775,6 +772,16 @@ class MATHMinerva_OLMES(MATHMinerva):
         return _OLMES_FEWSHOTS[: self.num_fewshot]
 
 
+class MATHMinerva_OLMES_NONL(MATHMinerva_OLMES):
+    NAME = "MATHMinerva_OLMES_NONL"
+
+    def __init__(self, num_fewshot: int = 4) -> None:
+        if num_fewshot != 4:
+            logger.warning("MATHMinerva_OLMES_NONL supports a fixed num_fewshot of 4.")
+        super().__init__(num_fewshot=4)
+        self.stop_sequences = ["Problem:"]
+
+
 class MATHMinervaBPB(MATHMinerva_OLMES):
     REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "MATHMinervaBPB"

{eval_framework-0.8.0 → eval_framework-0.8.2}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -10,6 +10,7 @@ from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglik
 from eval_framework.shared.types import BaseMetricContext
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
+from eval_framework.tasks.task_style import BPBStyle
 
 logger = logging.getLogger(__name__)
 
@@ -314,3 +315,90 @@ class MBPP_OLMES(MBPP):
         mbpp_ground_truth = str(sample.ground_truth)
         code = self._code_expander(extracted_code, mbpp_ground_truth)
         return code
+
+
+class MBPP_EvalPlus(MBPP):
+    """Version that faithfully follows the EvalPlus prompt format, which OLMES only approximated."""
+
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
+    NAME = "MBPP_EvalPlus"
+    FEWSHOT_SPLIT = "test"
+
+    def __init__(self, num_fewshot: int = 3) -> None:
+        super().__init__(num_fewshot)
+        if num_fewshot != 3:
+            logger.warning(f"MBPP_EvalPlus supports only 3-shot, got {num_fewshot}")
+        self.stop_sequences = ["```", "\nassert", "\nPlease provide"]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        text = item["text"] if "text" in item else item["prompt"]
+        test = item["test_list"][0]
+        return (
+            "Please provide a self-contained Python script that solves the following problem"
+            f" in a markdown code block:\n```\n{text.strip()}\n{test}\n```\n"
+        )
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return (
+            "Below is a Python script with a self-contained function that solves the problem"
+            " and passes corresponding tests:\n```python"
+        )
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        # The cue ends at ```python (no trailing newline, since it is stripped as the final eval
+        # message); the fewshot target adds the newline before the code explicitly.
+        return self._get_cue_text(item) + "\n" + item["code"] + "\n```"
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return list(_OLMES_FEWSHOT_EXAMPLES)
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample) -> str:  # type: ignore[override]
+        extracted_code = completion_text + "\n"
+        mbpp_ground_truth = str(sample.ground_truth)
+        code = self._code_expander(extracted_code, mbpp_ground_truth)
+        return code
+
+
+class MBPP_BPB_EvalPlus(BaseTask[str]):
+    """BPB (loglikelihood) counterpart of MBPP_EvalPlus."""
+
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
+    NAME = "MBPP_BPB_EvalPlus"
+    DATASET_PATH = "google-research-datasets/mbpp"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+    SUBJECTS = ["full"]
+    LANGUAGE = Language.ENG
+    TASK_STYLER = BPBStyle(
+        question_prefix="",
+        trailing_newline=False,
+        cue_text=(
+            "Below is a Python script with a self-contained function that solves the problem"
+            f" and passes corresponding tests:\n{BEGIN}"
+        ),
+        leading_space_continuations=False,
+    )
+
+    def __init__(self, num_fewshot: int = 3) -> None:
+        super().__init__(num_fewshot)
+        if num_fewshot != 3:
+            logger.warning(f"MBPP_BPB_EvalPlus supports only 3-shot, got {num_fewshot}")
+
+    def _get_raw_question(self, item: dict[str, Any]) -> str:
+        text = item["text"] if "text" in item else item["prompt"]
+        test = item["test_list"][0]
+        return (
+            "Please provide a self-contained Python script that solves the following problem"
+            f" in a markdown code block:\n```\n{text.strip()}\n{test}\n```\n"
+        )
+
+    def _get_choices(self, item: dict[str, Any]) -> list[str]:
+        return ["\n" + item["code"] + f"\n{END}"]
+
+    def _get_correct_index(self, item: dict[str, Any]) -> int:
+        return 0
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return list(_OLMES_FEWSHOT_EXAMPLES)

{eval_framework-0.8.0 → eval_framework-0.8.2}/src/eval_framework/tasks/task_names.py

@@ -36,6 +36,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEvalDe")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATH500")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES_NONL")
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalCpp")
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJava")
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJs")
@@ -50,6 +51,8 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPSh")
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPPBPB")
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_EvalPlus")
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_BPB_EvalPlus")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES")

{eval_framework-0.8.0 → eval_framework-0.8.2}/src/eval_framework/utils/logging.py

@@ -1,5 +1,4 @@
 import logging
-import sys
 from pathlib import Path
 
 VERBOSITY_MAP = {
@@ -16,7 +15,7 @@ def setup_logging(
     Set up centralized logging configuration for the entire framework.
 
     Args:
-        output_dir: Directory to save log files. If None, logs only to console.
+        output_dir: Directory to save log files. If None, no file handler is attached.
         log_level: Logging level (default: INFO)
         log_filename: Name of the log file
 
@@ -26,9 +25,6 @@ def setup_logging(
     # Map verbosity integer to logging level
     mapped_log_level = VERBOSITY_MAP.get(log_level, logging.INFO)
 
-    # Basic configuration
-    logging.basicConfig(level=mapped_log_level)
-
     # Create formatter
     formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
 
@@ -37,13 +33,7 @@ def setup_logging(
     root_logger.handlers.clear()
     root_logger.setLevel(mapped_log_level)
 
-    # Console handler (always present)
-    console_handler = logging.StreamHandler(sys.stdout)
-    console_handler.setLevel(mapped_log_level)
-    console_handler.setFormatter(formatter)
-    root_logger.addHandler(console_handler)
-
-    # File handler (if output directory provided)
+    # File handler (if output directory provided). No console handler: keeps stdout free for tqdm.
     if output_dir:
         output_dir.mkdir(parents=True, exist_ok=True)
         log_file = output_dir / log_filename
@@ -54,9 +44,5 @@ def setup_logging(
         root_logger.addHandler(file_handler)
 
         root_logger.info(f"Logging configured. File: {log_file}")
-    else:
-        root_logger.info("Logging configured (console only)")
-
-    root_logger.info(f"Output directory for logs: {output_dir if output_dir else 'None'}")
 
     return root_logger

eval_framework-0.8.2/src/eval_framework/utils/tqdm_handler.py

@@ -0,0 +1,7 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_disable_bar_flag() -> bool:
+    return logger.getEffectiveLevel() >= logging.WARNING

eval_framework-0.8.0/src/eval_framework/utils/tqdm_handler.py

@@ -1,14 +0,0 @@
-import logging
-
-from tqdm import tqdm
-
-logger = logging.getLogger(__name__)
-
-
-def safe_tqdm_write(msg: str, level: int = logging.INFO) -> None:
-    if logger.isEnabledFor(level):
-        tqdm.write(msg)
-
-
-def get_disable_bar_flag() -> bool:
-    return logger.getEffectiveLevel() >= logging.WARNING