eval-framework 0.8.7__tar.gz → 0.9.0__tar.gz

{eval_framework-0.8.7 → eval_framework-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.7
+Version: 0.9.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -216,17 +216,17 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.0,<4
+Requires-Dist: nltk>=3.10.1,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
 Requires-Dist: langdetect>=1.0.9,<2
 Requires-Dist: spacy>=3.8.14,<4
 Requires-Dist: jsonschema>=4.26.0,<5
-Requires-Dist: mysql-connector-python>=9.7.0,<10
+Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
-Requires-Dist: llm-sandbox[docker]==0.3.39
+Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.50.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.53.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.8.7 → eval_framework-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.7"
+version = "0.9.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -23,17 +23,17 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.0,<4",
+  "nltk>=3.10.1,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
   "langdetect>=1.0.9,<2", # required by the original ifeval implementation
   "spacy>=3.8.14,<4",
   "jsonschema>=4.26.0,<5",
-  "mysql-connector-python>=9.7.0,<10", # required for sql-related tasks
+  "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
   "sympy>=1.14.0,<2",
-  "llm-sandbox[docker]==0.3.39",
+  "llm-sandbox[docker]==0.3.44",
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.7.23",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.50.0,<3",
+  "openai>=2.53.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -92,7 +92,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260716,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.0",
+  "ruff>=0.16.1",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/response_generator.py

@@ -179,7 +179,7 @@ class ResponseGenerator:
         :param should_preempt_callable: function to check if preempt is called
         :return: list of responses, preempted
         """
-        logger.info(f"{RED}[ Running task {self.task.NAME} against model ------------ ]{RESET}")
+        logger.info(f"{RED}[ Running task {self.task.display_name()} against model ------------ ]{RESET}")
         self.start_time, monotonic_start = time.time(), time.monotonic()
         run_fn = self._generative_output_type_selector()
         self._verify_loaded_metadata_compatibility()

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/suite.py

@@ -18,7 +18,7 @@ from eval_framework.context.local import _load_model
 from eval_framework.result_processors.result_processor import generate_output_dir
 from eval_framework.run import _run_single_task
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.registry import is_registered
+from eval_framework.tasks.registry import registry
 
 logger = logging.getLogger(__name__)
 
@@ -108,7 +108,7 @@ class TaskSuite(BaseModel):
         if isinstance(self.tasks, str):
             if self.name is None:
                 self.name = self.tasks
-            if not is_registered(self.tasks):
+            if self.tasks not in registry():
                 raise ValueError(f"Task '{self.tasks}' is not registered.")
         elif not self.tasks:
             raise ValueError(f"TaskSuite '{self.name}': 'tasks' must not be empty.")

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/base.py

@@ -2,7 +2,7 @@ import logging
 import os
 import random
 import traceback
-from abc import ABC
+from abc import ABC, abstractmethod
 from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
@@ -86,7 +86,37 @@ SubjectType = TypeVar("SubjectType")
 logger = logging.getLogger(__name__)
 
 
-class BaseTask[SubjectType](ABC):
+class Task(ABC):
+    """The contract a caller relies on to run an evaluation"""
+
+    @abstractmethod
+    def iterate_samples(self, num_samples: int | None = None) -> Iterable[Sample]: ...
+
+    @abstractmethod
+    def generate_completions(
+        self,
+        llm: "BaseLLM",
+        samples: list[Sample],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        fail_on_error: bool = True,
+    ) -> list[Completion]:
+        """Run ``llm`` over ``samples`` and return their completions."""
+
+    @abstractmethod
+    def get_metadata(self) -> dict[str, str | list[str]]:
+        """Descriptive metadata about the eval for result reporting."""
+
+    @abstractmethod
+    def get_response_type(self) -> ResponseType: ...
+
+    @abstractmethod
+    def display_name(self) -> str:
+        """Human-readable display name. Is allowed to have special characters and whitespaces."""
+        ...
+
+
+class BaseTask[SubjectType](Task):
     NAME: str
     DATASET_PATH: str
     SAMPLE_SPLIT: str
@@ -497,3 +527,6 @@ class BaseTask[SubjectType](ABC):
     def METRICS(cls) -> list[type["BaseMetric"]]:
         """For backwards compatibility."""
         return cls.get_metrics()
+
+    def display_name(self) -> str:
+        return self.NAME

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -232,3 +232,8 @@ class GPQA_COT(GPQA):
         # index 1 selects the letter
         answer_key = choices[correct_answer_position][1]
         return answer_key
+
+
+class GPQA_DIAMOND_COT(GPQA_COT):
+    NAME = "GPQA_DIAMOND_COT"
+    SUBJECTS = ["gpqa_diamond"]

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/hf-dataset-revisions.json

@@ -23,7 +23,6 @@
     "google-research-datasets/nq_open": "5dd9790a83002ad084ddeb7c420dc716852c6f28",
     "google/IFEval": "966cd89545d6b6acfd7638bc708b98261ca58e84",
     "jzhang86/de_ifeval": "4f52d847003b3c83cc282e9d296853a24b34b19a",
-    "mandarjoshi/trivia_qa": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
     "math-ai/aime25": "563bb8404243c5f09de6ec262f2db674fe5bce9b",
     "math-ai/aime26": "79037aebdb6580008fb960d17cb21fd3099083e3",
     "nuprl/MultiPL-E": "28441b6024e71d4a1c1c0f6bf171c935cd5a43f2",

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/markdown_doc.py

@@ -47,7 +47,7 @@ def markdown_doc(
     buf.write(f"- Module: `{module}`\n\n")
 
     if http_path:
-        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
+        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n")
     else:
         assert example_messages is not None, "a task without a dataset link must supply an example sample"
         for split, size in (split_sizes or {}).items():

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/registry.py

@@ -18,7 +18,6 @@ __all__ = [
     "EvalFactory",
     "Registry",
     "with_registry",
-    "get_task",
     "is_registered",
     "validate_task_name",
     "registered_task_names",
@@ -36,8 +35,8 @@ class EvalFactory(ABC):
     """
 
     @abstractmethod
-    def task_class(self) -> type[BaseTask]:
-        """Return the task class, importing it on first access if necessary."""
+    def id(self) -> str:
+        "Canonical key used to register this benchmark"
 
     @property
     @abstractmethod
@@ -75,13 +74,10 @@ class EvalFactory(ABC):
         user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
+    @abstractmethod
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         """Render the eval's documentation as markdown."""
-        try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
-        except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
-        return task.markdown_doc(formatters)
+        ...
 
 
 class _Lazy(EvalFactory):
@@ -104,6 +100,9 @@ class _Lazy(EvalFactory):
     def source_module(self) -> str:
         return self._module
 
+    def id(self) -> str:
+        return self._class_name
+
     def task_class(self) -> type[BaseTask]:
         if self._loaded is None:
             module = importlib.import_module(self._module)
@@ -152,6 +151,13 @@ class _Lazy(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
+
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -163,8 +169,8 @@ class _Eager(EvalFactory):
     def source_module(self) -> str:
         return self._task.__module__
 
-    def task_class(self) -> type[BaseTask]:
-        return self._task
+    def id(self) -> str:
+        return self._task.__name__
 
     def create(
         self,
@@ -173,7 +179,7 @@ class _Eager(EvalFactory):
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
     ) -> BaseTask:
-        return self.task_class().with_overwrite(
+        return self._task.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
@@ -188,7 +194,7 @@ class _Eager(EvalFactory):
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
     ) -> BaseTask:
-        perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
+        perturbation_task_class = create_perturbation_class(self._task, perturbation_config)
         return perturbation_task_class.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
@@ -198,15 +204,22 @@ class _Eager(EvalFactory):
 
     def response_type(self) -> ResponseType:
         """The eval's response type"""
-        return self.task_class().get_response_type()
+        return self._task.get_response_type()
 
     def metrics(self) -> list[type["BaseMetric"]]:
         """The eval's metrics"""
-        return self.task_class().get_metrics()
+        return self._task.get_metrics()
 
     def display_name(self) -> str:
         """The eval's human-readable display name (the task's ``NAME``)."""
-        return self.task_class().NAME
+        return self._task.NAME
+
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
 
 
 class Registry:
@@ -255,10 +268,6 @@ class Registry:
 
         return factory
 
-    def add(self, task: type[BaseTask]) -> None:
-        task_key = self._task_key(task.NAME)
-        self._registry[task_key] = (task.NAME, _Eager(task))
-
     def __setitem__(self, name: str, factory: EvalFactory) -> None:
         task_key = self._task_key(name)
         if task_key in self._registry:
@@ -266,6 +275,24 @@ class Registry:
 
         self._registry[task_key] = (name, factory)
 
+    def register(self, task: type[BaseTask]) -> str:
+        """The class name is used as the task name."""
+        if not issubclass(task, BaseTask):
+            raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
+        name = task.__name__
+        self[name] = _Eager(task)
+        return name
+
+    def register_lazy(self, class_path: str, /) -> None:
+        """Register a task by its dotted class path, without importing its module."""
+        if "." not in class_path:
+            raise ValueError(
+                f"Invalid class path `{class_path}`. This needs to be a global path like "
+                "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
+            )
+        base_module, class_name = class_path.rsplit(".", maxsplit=1)
+        self[class_name] = _Lazy(class_name=class_name, module=base_module)
+
 
 _REGISTRY = Registry()
 
@@ -298,43 +325,16 @@ def is_registered(name: str, /) -> bool:
 
 def validate_task_name(name: str) -> str:
     """Pydantic-style validator for task names."""
-    if not is_registered(name):
+    if name not in registry():
         raise ValueError(f"Task not registered: {name}")
     return name
 
 
-def get_task(name: str, /) -> type[BaseTask]:
-    """Return a registered task for a given name.
-
-    Note: This method will import any lazily registered task.
-    """
-    return _REGISTRY[name].task_class()
-
-
 def register_task(task: type[BaseTask]) -> str:
     """The class name is used as the task name."""
-    if not issubclass(task, BaseTask):
-        raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
-    name = task.__name__
-    _REGISTRY[name] = _Eager(task)
-    return name
+    return registry().register(task)
 
 
 def register_lazy_task(class_path: str, /) -> None:
-    """Register a task without importing it.
-
-    Lazily register a task without importing the module.
-
-    Args:
-        class_path: The full path to the task class. For example,
-            `eval_framework.tasks.benchmarks.mmlu.MMLU`.
-        extras: Any extra dependencies of `eval_framework` that need to be installed for this task.
-    """
-    if "." not in class_path:
-        raise ValueError(
-            f"Invalid class path `{class_path}`. This needs to be a global path like "
-            "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
-        )
-
-    base_module, class_name = class_path.rsplit(".", maxsplit=1)
-    _REGISTRY[class_name] = _Lazy(class_name=class_name, module=base_module)
+    """Register a task by its dotted class path, without importing its module."""
+    registry().register_lazy(class_path)

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/task_loader.py

@@ -8,7 +8,8 @@ from types import ModuleType
 from typing import Any
 
 from eval_framework.tasks.base import BaseTask
-from eval_framework.tasks.registry import is_registered, register_task
+from eval_framework.tasks.registry import Registry
+from eval_framework.tasks.registry import registry as global_registry
 
 logger = logging.getLogger(__name__)
 
@@ -46,14 +47,13 @@ def import_file(f: str | os.PathLike, /) -> Any:
     return user_module
 
 
-def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
+def load_extra_tasks(module_paths: Sequence[str | os.PathLike], registry: Registry | None = None) -> None:
     """Dynamically load and register user-defined tasks from a list of files or directories.
 
-    Each .py file found will be imported, and any BaseTask subclass will be registered
-    in the TaskName enum for use by name.
-    Provides clear error messages for missing/invalid files or import errors.
+    Each .py file found is imported, and any BaseTask subclass is registered
     """
     assert not (isinstance(module_paths, str)), "module_paths must be a sequence of strings / os.PathLike objects"
+    registry = registry if registry is not None else global_registry()
     for file_path in find_all_python_files(*module_paths):
         user_module = import_file(file_path)
 
@@ -63,7 +63,7 @@ def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
                 if not hasattr(obj, "NAME"):
                     logger.info(f"[User Task Loader] Skipping {obj.__module__} - no NAME attribute present.")
                 else:
-                    if is_registered(obj.NAME):
+                    if obj.NAME in registry:
                         # two classes with the same NAME attribute
                         logger.info(obj.__module__)
 
@@ -77,5 +77,5 @@ def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
                     else:
                         # if there is no duplicate name conflict then register the new task
                         class_obj = getattr(user_module, name)
-                        register_task(class_obj)
+                        registry.register(class_obj)
                         logger.info(f"[User Task Loader] Registered task: {class_obj.NAME}")

{eval_framework-0.8.7 → eval_framework-0.9.0}/src/eval_framework/tasks/task_names.py

@@ -24,6 +24,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG")
     register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT")
     register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8K_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8KBPB")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinervaBPB")
@@ -71,8 +72,6 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT")
-    register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA")
-    register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TriviaQA_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze")
     register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES")
@@ -81,9 +80,3 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES")
-    try:
-        # Importing the companion registers the additional tasks from the module.
-        # This is mostly for convenience for internal use-cases
-        import eval_framework_companion  # noqa
-    except ImportError:
-        pass

eval_framework-0.8.7/src/eval_framework/tasks/benchmarks/triviaqa.py

@@ -1,78 +0,0 @@
-import random
-from typing import Any
-
-from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
-from eval_framework.metrics.completion.f1 import F1, F1SquadNormalized
-from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
-from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
-
-
-class TRIVIAQA(BaseTask[str]):
-    """Trivia QA dataset: https://huggingface.co/datasets/mandarjoshi/trivia_qa"""
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "TriviaQA"
-    DATASET_PATH = "mandarjoshi/trivia_qa"
-    SAMPLE_SPLIT = "validation"
-    FEWSHOT_SPLIT = "train"
-    RESPONSE_TYPE = ResponseType.COMPLETION
-    METRICS = [AccuracyCompletion, F1]
-    SUBJECTS = ["rc.wikipedia.nocontext"]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
-    LANGUAGE = Language.ENG
-
-    def __init__(self, num_fewshot: int = 0) -> None:
-        super().__init__(num_fewshot)
-        self.stop_sequences = ["\n"]
-        self.max_tokens = 400  # the max length of the ground truth is 282 characters while the average is ~16
-        self.rnd_choice_shuffle = random.Random()
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        prompt = f"Question: {item['question'].strip()}\nAnswer:"
-        return prompt
-
-    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        target = self._get_ground_truth(item)[0]
-        assert target is not None
-        assert isinstance(target, str)
-        return f" {target}"
-
-    def _get_ground_truth(self, item: dict[str, Any]) -> list[str]:
-        return item["answer"]["aliases"]
-
-    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
-        return completion_text.strip().rstrip(".")
-
-
-class TriviaQA_MA(TRIVIAQA):
-    """TriviaQA with the exact system prompt used in MA training"""
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "TriviaQA_MA"
-    SUBJECTS = ["rc.wikipedia"]
-    UNANSWERABLE_STR = "unanswerable"
-
-    METRICS = [AccuracyCompletion, F1, F1SquadNormalized]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer", "Context", "unanswerable"]
-
-    def __init__(self, num_fewshot: int = 0) -> None:
-        super().__init__(num_fewshot)
-        self.stop_sequences = []
-        self.max_tokens = 27_000
-
-    def _get_context_text(self, item: dict[str, Any]) -> str:
-        return "\n\n".join(item["entity_pages"]["wiki_context"])
-
-    def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
-        return (
-            "You are a helpful assistant and will answer the user's questions carefully, "
-            "logically, accurately and well-reasoned.\n"
-            "Use the given context to answer the question faithfully. Answer only if the "
-            f"answer is present in the given context, otherwise respond with '{self.UNANSWERABLE_STR}' "
-            "if the answer is not present in the context."
-        )
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        return f"Context:\n{self._get_context_text(item)}\n\nQuestion:\n{item['question'].strip()}\n"