eval-framework 0.6.3__tar.gz → 0.6.4__tar.gz

{eval_framework-0.6.3 → eval_framework-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.6.3
+Version: 0.6.4
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.6.3 → eval_framework-0.6.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.6.3"
+version = "0.6.4"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/context/determined.py

@@ -30,6 +30,7 @@ class TaskArgs(BaseModel):
     judge_model_args: dict[str, Any] = {}
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
     perturbation_config: PerturbationConfig | None = None
     repeats: int | None = None
 
@@ -108,6 +109,7 @@ class DeterminedContext(EvalContext):
             "task_subjects",
             "batch_size",
             "hf_revision",
+            "user_prompt_suffix",
             "judge_model_name",
             "judge_model_args",
             "perturbation_config",
@@ -121,6 +123,9 @@ class DeterminedContext(EvalContext):
         # Hyperparameters take precedence over core context
         llm_name = self.hparams.llm_name or self.llm_name
         judge_model_name = self.hparams.task_args.judge_model_name or self.judge_model_name
+        user_prompt_suffix = self.hparams.task_args.user_prompt_suffix
+        if user_prompt_suffix is None:
+            user_prompt_suffix = self.user_prompt_suffix
 
         llm_class = _load_model(llm_name, models_path=self.models_path)
         llm_judge_class: type[BaseLLM] | None = (
@@ -139,6 +144,7 @@ class DeterminedContext(EvalContext):
             task_name=self.hparams.task_args.task_name,
             task_subjects=self.hparams.task_args.task_subjects,
             hf_revision=self.hparams.task_args.hf_revision or self.hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
             perturbation_config=self.hparams.task_args.perturbation_config or self.perturbation_config,
             output_dir=self.hparams.output_dir,
             llm_judge_class=llm_judge_class,

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/context/eval.py

@@ -57,6 +57,7 @@ class EvalContext(AbstractContextManager):
         task_name: str | None = None,
         task_subjects: list[str] | None = None,
         hf_revision: str | None = None,
+        user_prompt_suffix: str | None = None,
         output_dir: Path | None = None,
         wandb_project: str | None = None,
         wandb_entity: str | None = None,
@@ -86,6 +87,7 @@ class EvalContext(AbstractContextManager):
         self.task_name = task_name
         self.task_subjects = task_subjects
         self.hf_revision = hf_revision
+        self.user_prompt_suffix = user_prompt_suffix
         self.output_dir = output_dir
         self.wandb_project = wandb_project
         self.wandb_entity = wandb_entity

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/context/local.py

@@ -52,6 +52,7 @@ class LocalContext(EvalContext):
             task_name=self.task_name,
             task_subjects=self.task_subjects,
             hf_revision=self.hf_revision,
+            user_prompt_suffix=self.user_prompt_suffix,
             output_dir=self.output_dir,
             hf_upload_dir=self.hf_upload_dir,
             hf_upload_repo=self.hf_upload_repo,

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/response_generator.py

@@ -66,10 +66,14 @@ class ResponseGenerator:
                 config.num_fewshot,
                 config.task_subjects,
                 config.hf_revision,
+                user_prompt_suffix=config.user_prompt_suffix,
             )
         else:
             self.task = registry()[config.task_name].create(
-                config.num_fewshot, config.task_subjects, config.hf_revision
+                config.num_fewshot,
+                config.task_subjects,
+                config.hf_revision,
+                user_prompt_suffix=config.user_prompt_suffix,
             )
 
         self.response_type = self.task.get_response_type()
@@ -345,9 +349,10 @@ class ResponseGenerator:
             "llm_args",
             "perturbation_config",
             "repeats",
+            "user_prompt_suffix",
         ]
         for key in keys:
-            if loaded_metadata[key] != current_metadata[key]:
+            if loaded_metadata.get(key) != current_metadata[key]:
                 raise ValueError(f"Existing metadata does not match current metadata for {key}.")
 
     def __del__(self) -> None:

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/run.py

@@ -150,6 +150,13 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="A tag name, a branch name, or commit hash for the task HF dataset.",
     )
+    parser.add_argument(
+        "--user-prompt-suffix",
+        type=str,
+        required=False,
+        default=None,
+        help="Text appended verbatim to the evaluated user prompt before formatting.",
+    )
     parser.add_argument(
         "--judge-models",
         type=Path,
@@ -343,6 +350,7 @@ def _run_single_task(kwargs: dict) -> None:
         task_name=kwargs["task_name"],
         task_subjects=kwargs["task_subjects"],
         hf_revision=kwargs["hf_revision"],
+        user_prompt_suffix=kwargs.get("user_prompt_suffix"),
         output_dir=kwargs["output_dir"],
         wandb_project=kwargs["wandb_project"],
         wandb_entity=kwargs["wandb_entity"],

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/suite.py

@@ -34,6 +34,7 @@ _EVAL_CONFIG_FIELDS = {
     "batch_size",
     "task_subjects",
     "hf_revision",
+    "user_prompt_suffix",
 }
 
 _HYPERPARAM_FIELDS = _LLM_ARG_FIELDS | _EVAL_CONFIG_FIELDS
@@ -100,6 +101,7 @@ class TaskSuite(BaseModel):
     batch_size: int | None = None
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
 
     @model_validator(mode="after")
     def validate_suite(self) -> Self:

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/tasks/base.py

@@ -115,6 +115,7 @@ class BaseTask[SubjectType](ABC):
 
     def __init__(self, num_fewshot: int = 0) -> None:
         self.num_fewshot = num_fewshot
+        self.user_prompt_suffix: str | None = None
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
         self._apply_hf_revision()
@@ -129,9 +130,17 @@ class BaseTask[SubjectType](ABC):
 
     @classmethod
     def with_overwrite(
-        cls, num_fewshot: int, *, custom_subjects: list[str] | None, custom_hf_revision: str | None
+        cls,
+        num_fewshot: int,
+        *,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> Self:
         instance = cls(num_fewshot=num_fewshot)
+        if user_prompt_suffix is not None and instance.get_response_type() != ResponseType.COMPLETION:
+            raise ValueError("user_prompt_suffix is only supported for completion tasks.")
+        instance.user_prompt_suffix = user_prompt_suffix
 
         # If custom subjects were provided during initialization, they take precedence over the class-level SUBJECTS.
         filtered_subjects = instance._filter_task_subjects(custom_subjects=custom_subjects)
@@ -241,7 +250,7 @@ class BaseTask[SubjectType](ABC):
 
     def _get_messages(self, item: dict[str, Any]) -> list[Message]:
         example_messages = self._get_example_messages(item)
-        instruction_message = self._get_instruction_messages(item)
+        instruction_message = self._apply_user_prompt_suffix(self._get_instruction_messages(item))
         cue_text = self._get_cue_text(item)
         cue_message = [Message(role=Role.ASSISTANT, content=cue_text)] if cue_text else []
         messages = example_messages + instruction_message + cue_message
@@ -254,6 +263,18 @@ class BaseTask[SubjectType](ABC):
             return [Message(role=Role.SYSTEM, content=system_prompt_text)] + messages
         return messages
 
+    def _apply_user_prompt_suffix(self, instruction_messages: list[Message]) -> list[Message]:
+        """Append the configured suffix verbatim to the evaluated user turn."""
+        if self.user_prompt_suffix is None:
+            return instruction_messages
+
+        for message in reversed(instruction_messages):
+            if message.role == Role.USER:
+                message.content = f"{message.content}{self.user_prompt_suffix}"
+                return instruction_messages
+
+        raise ValueError("Cannot append user_prompt_suffix: evaluated instruction contains no user message.")
+
     def _get_instruction_messages(self, item: dict[str, Any]) -> list[Message]:
         return [Message(role=Role.USER, content=self._get_instruction_text(item))]
 

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/tasks/eval_config.py

@@ -47,6 +47,7 @@ class EvalConfig(BaseConfig):
     task_name: Annotated[str, AfterValidator(validate_task_name)]
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
     llm_class: type[BaseLLM]
     llm_args: dict[str, Any] = Field(default_factory=dict)
     llm_judge_class: type[BaseLLM] | None = None
@@ -141,4 +142,7 @@ class EvalConfig(BaseConfig):
 
     def model_json_robust_subset_dump(self) -> str:
         model_dump = self.model_dump(mode="json", exclude=KEYS_UNRELATED_TO_RESULTS)
+        # Preserve hashes for evaluations created before this optional field existed.
+        if self.user_prompt_suffix is None:
+            model_dump.pop("user_prompt_suffix", None)
         return json.dumps(model_dump, sort_keys=True)

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/tasks/registry.py

@@ -62,7 +62,11 @@ class EvalFactory(ABC):
 
     @abstractmethod
     def create(
-        self, num_fewshot: int, custom_subjects: list[str] | None, custom_hf_revision: str | None
+        self,
+        num_fewshot: int,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
     @abstractmethod
@@ -72,6 +76,7 @@ class EvalFactory(ABC):
         num_fewshot: int,
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
 
@@ -101,9 +106,18 @@ class _Lazy(EvalFactory):
             self._loaded = getattr(module, self._class_name)
         return self._loaded
 
-    def create(self, num_fewshot: int, custom_subjects: list[str] | None, custom_hf_revision: str | None) -> BaseTask:
+    def create(
+        self,
+        num_fewshot: int,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
+    ) -> BaseTask:
         return self.task_class().with_overwrite(
-            num_fewshot=num_fewshot, custom_subjects=custom_subjects, custom_hf_revision=custom_hf_revision
+            num_fewshot=num_fewshot,
+            custom_subjects=custom_subjects,
+            custom_hf_revision=custom_hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
         )
 
     def create_perturbation(
@@ -112,12 +126,14 @@ class _Lazy(EvalFactory):
         num_fewshot: int,
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> BaseTask:
         perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
         return perturbation_task_class.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
         )
 
     def response_type(self) -> ResponseType:
@@ -149,9 +165,18 @@ class _Eager(EvalFactory):
     def task_class(self) -> type[BaseTask]:
         return self._task
 
-    def create(self, num_fewshot: int, custom_subjects: list[str] | None, custom_hf_revision: str | None) -> BaseTask:
+    def create(
+        self,
+        num_fewshot: int,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
+    ) -> BaseTask:
         return self.task_class().with_overwrite(
-            num_fewshot=num_fewshot, custom_subjects=custom_subjects, custom_hf_revision=custom_hf_revision
+            num_fewshot=num_fewshot,
+            custom_subjects=custom_subjects,
+            custom_hf_revision=custom_hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
         )
 
     def create_perturbation(
@@ -160,12 +185,14 @@ class _Eager(EvalFactory):
         num_fewshot: int,
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> BaseTask:
         perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
         return perturbation_task_class.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
         )
 
     def response_type(self) -> ResponseType:

{eval_framework-0.6.3 → eval_framework-0.6.4}/src/eval_framework/tasks/task_names.py

@@ -63,6 +63,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.FullTextMMLU")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_OLMES")