eval-framework 0.9.2__tar.gz → 0.10.4__tar.gz

{eval_framework-0.9.2 → eval_framework-0.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.9.2
+Version: 0.10.4
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -208,6 +208,8 @@ Classifier: Operating System :: OS Independent
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
@@ -216,12 +218,11 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.2,<4
+Requires-Dist: nltk>=3.10.3,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
 Requires-Dist: langdetect>=1.0.9,<2
-Requires-Dist: spacy>=3.8.14,<4
 Requires-Dist: jsonschema>=4.26.0,<5
 Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
@@ -230,9 +231,9 @@ Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
-Requires-Dist: wandb>=0.28.1,<1
-Requires-Dist: boto3>=1.43.19,<2
-Requires-Dist: numpy>=2.5.1
+Requires-Dist: wandb>=0.28.2,<1
+Requires-Dist: boto3>=1.43.72,<2
+Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
@@ -240,7 +241,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.53.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=3.1.0,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -248,7 +249,7 @@ Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.13.0,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
-Requires-Python: >=3.12, <3.13
+Requires-Python: >=3.12, <3.15
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework
 Provides-Extra: accelerate
 Provides-Extra: all
@@ -314,8 +315,6 @@ There are optional extras available to unlock specific features of the library:
 
 As a short hand, the `all` extra installs all of the above.
 
-For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
-
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

{eval_framework-0.9.2 → eval_framework-0.10.4}/README.md

@@ -53,8 +53,6 @@ There are optional extras available to unlock specific features of the library:
 
 As a short hand, the `all` extra installs all of the above.
 
-For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
-
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

eval_framework-0.10.4/pyproject.toml

@@ -0,0 +1,162 @@
+[project]
+name = "eval-framework"
+version = "0.10.4"
+description = "Evaluation Framework"
+readme = "README.md"
+requires-python = ">=3.12,<3.15"
+classifiers = [
+    "Operating System :: OS Independent",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Software Development :: Libraries",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pyyaml>=6.0.3,<7",
+    "xmltodict>=1.0.4,<1.1",
+    "pydantic>=2.13.4,<3",
+    "datasets>=5.0.1,<6",
+    "pycountry>=26.2.16,<27",
+    "nltk>=3.10.3,<4",
+    "python-dotenv>=1.2.2,<2",
+    "lingua-language-detector>=2.2.0,<3",
+    "google-crc32c>=1.8.0,<2",
+    "langdetect>=1.0.9,<2",
+    "jsonschema>=4.26.0,<5",
+    "mysql-connector-python>=26.7.0,<27",
+    "psycopg2-binary>=2.9.12,<3",
+    "sympy>=1.14.0,<2",
+    "llm-sandbox[docker]==0.3.44",
+    "jsonlines>=4,<5",
+    "lxml>=6.1.1,<7",
+    "python-iso639>=2026.7.23",
+    "wandb>=0.28.2,<1",
+    "boto3>=1.43.72,<2",
+    "numpy>=2.5.2",
+    "antlr4-python3-runtime==4.11.0",
+    "scipy>=1.18.0,<2",
+]
+
+[project.license]
+file = "LICENSE"
+
+[[project.authors]]
+name = "Aleph Alpha Research"
+
+[project.optional-dependencies]
+determined = [
+    "determined>=0.38.1,<0.39",
+    "tensorboard==2.21.0",
+]
+api = ["aleph-alpha-client>=11.5.1"]
+openai = [
+    "openai>=3.1.0,<4",
+    "tiktoken>=0.13.0,<1",
+    "transformers>=4.45.2,<5",
+]
+transformers = [
+    "transformers>=4.45.2,<5",
+    "torch>=2.13.0,<3",
+    "accelerate>=1.14.0,<2",
+]
+accelerate = ["accelerate"]
+optional = [
+    "transformers>=4.45.2,<5",
+    "jinja2>=3.1.6,<4",
+]
+all = ["eval_framework[determined,api,openai,transformers,accelerate,optional]"]
+
+[project.urls]
+repository = "https://github.com/Aleph-Alpha-Research/eval-framework"
+
+[project.scripts]
+eval_framework = "eval_framework.run:run"
+
+[dependency-groups]
+dev = [
+    "mypy>=2.3.0,<3",
+    "pytest>=9.1.1,<10",
+    "pytest-mock>=3.15.1",
+    "pytest-xdist>=3.8.0,<4",
+    "pytest-sugar>1.1,<2",
+    "types-pyyaml>=6.0.12.20260724,<7",
+    "types-python-dateutil>=2.9.0.20260807,<3",
+    "types-requests>=2.33.0.20260712,<3",
+    "plotly>=6.9.0,<7",
+    "ruff>=0.16.3",
+    "pip-licenses>=5.5.5",
+]
+flash-attn = [
+    "flash-attn>=2.8.3.post1,<2.9",
+    "torch",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.12.6"]
+build-backend = "uv_build"
+
+[tool.uv]
+override-dependencies = ["requests>=2.32,<3"]
+
+[tool.uv.build-backend]
+module-name = [
+    "eval_framework",
+    "template_formatting",
+]
+
+[[tool.uv.extra-build-dependencies.flash-attn]]
+requirement = "torch"
+match-runtime = true
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "UP",
+    "I",
+]
+
+[tool.ruff.lint.isort]
+known-third-party = ["wandb"]
+
+[tool.ruff.lint.extend-per-file-ignores]
+"__init__.py" = ["F401"]
+"tests/tests_eval_framework/tasks/benchmarks/test_csqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gsm8k_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hendrycks_math_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_mbpp_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_piqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_siqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_winogrande_ellamind.py" = ["E501"]
+
+[tool.mypy]
+plugins = "pydantic.mypy"
+disallow_untyped_defs = true
+ignore_missing_imports = true
+files = [
+    "src",
+    "utils",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["./tests"]
+minversion = 8.0
+addopts = "-p 'no:legacypath' --doctest-modules -n auto"
+markers = [
+    "gpu: needs a GPU runner, otherwise test can not be run",
+    "cpu_slow: runs for a long time (on CPU)",
+    "slow_download: smoke tests that download large datasets (>15s); excluded from CI, run manually with -m slow_download",
+    "external_api: needs external services for execution",
+    "formatter_hash: formatter consistency tests using hash comparisons",
+]
+filterwarnings = ["ignore::DeprecationWarning:datasets.utils._dill:"]
+env = ["WANDB_MODE = disabled"]

eval_framework-0.9.2/pyproject.toml → eval_framework-0.10.4/pyproject.toml.orig

@@ -1,10 +1,10 @@
 [project]
 name = "eval-framework"
-version = "0.9.2"
+version = "0.10.4"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
-requires-python = ">=3.12,<3.13"
+requires-python = ">=3.12,<3.15"
 authors = [
   { name = "Aleph Alpha Research" }
 ]
@@ -13,6 +13,8 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
   "Programming Language :: Python :: 3 :: Only",
   "Topic :: Software Development :: Libraries",
   "Typing :: Typed",
@@ -23,12 +25,11 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.2,<4",
+  "nltk>=3.10.3,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
   "langdetect>=1.0.9,<2", # required by the original ifeval implementation
-  "spacy>=3.8.14,<4",
   "jsonschema>=4.26.0,<5",
   "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
@@ -37,9 +38,9 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.7.23",
-  "wandb>=0.28.1,<1",
-  "boto3>=1.43.19,<2",
-  "numpy>=2.5.1",
+  "wandb>=0.28.2,<1",
+  "boto3>=1.43.72,<2",
+  "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
   "antlr4-python3-runtime==4.11.0",
@@ -54,7 +55,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.53.0,<3",
+  "openai>=3.1.0,<4",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -89,10 +90,10 @@ dev = [
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
   "types-pyyaml>=6.0.12.20260724,<7",
-  "types-python-dateutil>=2.9.0.20260716,<3",
+  "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.1",
+  "ruff>=0.16.3",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -101,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.32,<0.11.33"]
+requires = ["uv_build>=0.12.5,<0.12.6"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]
@@ -137,6 +138,16 @@ known-third-party = ["wandb"]
 
 [tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = ["F401"]
+# Prompt strings are too long for:
+"tests/tests_eval_framework/tasks/benchmarks/test_csqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gsm8k_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hendrycks_math_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_mbpp_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_piqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_siqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_winogrande_ellamind.py" = ["E501"]
 
 [tool.mypy]
 plugins = "pydantic.mypy"
@@ -147,7 +158,7 @@ files = ["src", "utils"]
 [tool.pytest.ini_options]
 testpaths = ["./tests"]
 minversion = 8.0
-addopts = "-p 'no:legacypath' --doctest-modules"
+addopts = "-p 'no:legacypath' --doctest-modules -n auto"
 markers = [
     "gpu: needs a GPU runner, otherwise test can not be run",
     "cpu_slow: runs for a long time (on CPU)",

{eval_framework-0.9.2 → eval_framework-0.10.4}/src/eval_framework/context/determined.py

@@ -12,7 +12,6 @@ from eval_framework.context.eval import EvalContext
 from eval_framework.context.local import _load_model
 from eval_framework.llm.base import BaseLLM
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.perturbation import PerturbationConfig
 from eval_framework.tasks.registry import validate_task_name
 from eval_framework.tasks.task_loader import load_extra_tasks
 
@@ -31,7 +30,6 @@ class TaskArgs(BaseModel):
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
     user_prompt_suffix: str | None = None
-    perturbation_config: PerturbationConfig | None = None
     repeats: int | None = None
 
 
@@ -48,7 +46,7 @@ class Hyperparameters(BaseModel):
     description: str | None = None
     task_args: TaskArgs
     llm_args: dict[str, Any] | None = {}
-    extra_task_modules: list[str] | None = None
+    extra_tasks_dir: str | None = None
     delete_output_dir_after_upload: bool | None = None
 
 
@@ -67,19 +65,18 @@ class DeterminedContext(EvalContext):
             raise RuntimeError("Failed to retrieve cluster info.")
 
         # Load extra tasks if specified first
-        extra_task_modules = info.trial.hparams.get("extra_task_modules", None)
-        if extra_task_modules:
-            name = "extra_task_modules"
+        extra_tasks_dir = info.trial.hparams.get("extra_tasks_dir", None)
+        if extra_tasks_dir:
+            name = "extra_tasks_dir"
             val_cli = getattr(self, name, None)
-            val_hparams = extra_task_modules
-            if val_hparams:
-                if val_cli and val_hparams and val_cli != val_hparams:
-                    logger.info(
-                        f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters:"
-                        f"({val_hparams}). If it fails due to duplicate task names, remove the CLI argument and"
-                        "consolidate as a determined hyperparameter instead."
-                    )
-                load_extra_tasks(val_hparams)
+            val_hparams = extra_tasks_dir
+            if val_cli and val_hparams and val_cli != val_hparams:
+                logger.info(
+                    f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters:"
+                    f"({val_hparams}). If it fails due to duplicate task names, remove the CLI argument and"
+                    "consolidate as a determined hyperparameter instead."
+                )
+            load_extra_tasks(val_hparams)
 
         self.hparams = Hyperparameters(**info.trial.hparams)
 
@@ -112,7 +109,6 @@ class DeterminedContext(EvalContext):
             "user_prompt_suffix",
             "judge_model_name",
             "judge_model_args",
-            "perturbation_config",
             "repeats",
         ]:
             val_cli = getattr(self, name, None)
@@ -145,7 +141,6 @@ class DeterminedContext(EvalContext):
             task_subjects=self.hparams.task_args.task_subjects,
             hf_revision=self.hparams.task_args.hf_revision or self.hf_revision,
             user_prompt_suffix=user_prompt_suffix,
-            perturbation_config=self.hparams.task_args.perturbation_config or self.perturbation_config,
             output_dir=self.hparams.output_dir,
             llm_judge_class=llm_judge_class,
             judge_model_args=self.hparams.task_args.judge_model_args or self.judge_model_args,

{eval_framework-0.9.2 → eval_framework-0.10.4}/src/eval_framework/context/eval.py

@@ -9,7 +9,6 @@ from typing import Any
 import eval_framework
 from eval_framework.llm.base import BaseLLM
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.perturbation import PerturbationConfig
 
 
 def import_models(models_file: PathLike | str) -> dict[str, type[BaseLLM]]:
@@ -72,9 +71,6 @@ class EvalContext(AbstractContextManager):
         judge_model_args: dict[str, Any] | None = None,
         batch_size: int | None = None,
         description: str | None = None,
-        perturbation_type: str | None = None,
-        perturbation_probability: float | None = None,
-        perturbation_seed: int | None = None,
         randomize_judge_order: bool = False,
         delete_output_dir_after_upload: bool | None = None,
         repeats: int | None = None,
@@ -105,17 +101,6 @@ class EvalContext(AbstractContextManager):
         self.randomize_judge_order = randomize_judge_order
         self.delete_output_dir_after_upload = delete_output_dir_after_upload
         self.repeats = repeats
-        if perturbation_type or perturbation_probability is not None:
-            perturbation = {
-                "type": perturbation_type,
-                "probability": perturbation_probability,
-                "seed": perturbation_seed,
-            }
-            self.perturbation_config: PerturbationConfig | None = PerturbationConfig(
-                **{k: v for k, v in perturbation.items() if v is not None}
-            )
-        else:
-            self.perturbation_config = None
 
         self.config: EvalConfig | None = None
 

{eval_framework-0.9.2 → eval_framework-0.10.4}/src/eval_framework/context/local.py

@@ -48,7 +48,6 @@ class LocalContext(EvalContext):
             num_samples=self.num_samples,
             max_tokens=self.max_tokens,
             num_fewshot=self.num_fewshot,
-            perturbation_config=self.perturbation_config,
             task_name=self.task_name,
             task_subjects=self.task_subjects,
             hf_revision=self.hf_revision,

{eval_framework-0.9.2 → eval_framework-0.10.4}/src/eval_framework/response_generator.py

@@ -60,23 +60,13 @@ class ResponseGenerator:
         self.num_samples = config.num_samples
         self.save_intermediate_results = config.save_intermediate_results
 
-        if config.perturbation_config is not None:
-            self.task = registry()[config.task_name].create_perturbation(
-                config.perturbation_config,
-                config.num_fewshot,
-                config.task_subjects,
-                config.hf_revision,
-                user_prompt_suffix=config.user_prompt_suffix,
-                seed=RANDOM_SEED,
-            )
-        else:
-            self.task = registry()[config.task_name].create(
-                config.num_fewshot,
-                config.task_subjects,
-                config.hf_revision,
-                user_prompt_suffix=config.user_prompt_suffix,
-                seed=RANDOM_SEED,
-            )
+        self.task = registry()[config.task_name].create(
+            config.num_fewshot,
+            config.task_subjects,
+            config.hf_revision,
+            user_prompt_suffix=config.user_prompt_suffix,
+            seed=RANDOM_SEED,
+        )
 
         self.response_type = self.task.get_response_type()
 
@@ -341,7 +331,6 @@ class ResponseGenerator:
             "num_samples",
             "llm_name",
             "llm_args",
-            "perturbation_config",
             "repeats",
             "user_prompt_suffix",
         ]

{eval_framework-0.9.2 → eval_framework-0.10.4}/src/eval_framework/run.py

@@ -42,11 +42,11 @@ def parse_args() -> argparse.Namespace:
         help="The path to the Python module file containing model classes.",
     )
     parser.add_argument(
-        "--extra-task-modules",
-        nargs="*",
-        default=[],
+        "--extra-tasks-dir",
+        type=Path,
+        default=None,
         required=False,
-        help="List of files and folders containing additional task definitions.",
+        help="Directory containing additional task plugin modules to load.",
     )
     parser.add_argument(
         "--llm-name",
@@ -102,37 +102,6 @@ def parse_args() -> argparse.Namespace:
         help="Randomize the order of answers presented to the LLM judge to mitigate position bias.",
     )
 
-    # Perturbation arguments
-    parser.add_argument(
-        "--perturbation-type",
-        type=str,
-        required=False,
-        choices=[
-            "editor",
-            "permute",
-            "replace",
-            "delete",
-            "uppercase",
-        ],
-        help=(
-            "The type of perturbation to apply. Note that this may not make sense for some prompts, for example those "
-            "containing math and code."
-        ),
-    )
-    parser.add_argument(
-        "--perturbation-probability",
-        type=float,
-        required=False,
-        default=None,
-        help="The probability of applying a perturbation to each word or character (between 0.0 and 1.0).",
-    )
-    parser.add_argument(
-        "--perturbation-seed",
-        type=int,
-        required=False,
-        default=42,
-        help="Random seed controlling perturbations.",
-    )
     parser.add_argument(
         "--task-subjects",
         type=str,
@@ -313,12 +282,6 @@ def parse_args() -> argparse.Namespace:
 
     args.judge_model_args = judge_model_args
 
-    # if args.extra_task_modules:
-    #     # Convert the comma-separated string into a list
-    #     args.extra_task_modules = [file_or_dir.strip() for file_or_dir in args.extra_task_modules.split(",")]
-    # else:
-    #     args.extra_task_modules = None
-
     if args.task_suite and args.task_name:
         parser.error("--task-suite and --task-name are mutually exclusive.")
 
@@ -336,8 +299,8 @@ def _run_single_task(kwargs: dict) -> None:
     now = datetime.datetime.now()
     logger.info(f"starting time: {now}")
 
-    if kwargs.get("extra_task_modules"):
-        load_extra_tasks(kwargs["extra_task_modules"])
+    if kwargs.get("extra_tasks_dir"):
+        load_extra_tasks(kwargs["extra_tasks_dir"])
 
     context_name = kwargs.pop("context")
 
@@ -366,9 +329,6 @@ def _run_single_task(kwargs: dict) -> None:
         judge_model_args=kwargs["judge_model_args"],
         batch_size=kwargs["batch_size"],
         description=kwargs["description"],
-        perturbation_type=kwargs["perturbation_type"],
-        perturbation_probability=kwargs["perturbation_probability"],
-        perturbation_seed=kwargs["perturbation_seed"],
         randomize_judge_order=kwargs["randomize_judge_order"],
         delete_output_dir_after_upload=kwargs["delete_output_dir_after_upload"],
         # save_logs=kwargs["save_logs"],

{eval_framework-0.9.2 → eval_framework-0.10.4}/src/eval_framework/run_direct.py

@@ -32,9 +32,6 @@ if __name__ == "__main__":
         "hf_upload_dir": "",
         "description": "",
         "batch_size": 1,
-        "perturbation_type": None,
-        "perturbation_probability": None,
-        "perturbation_seed": None,
         "save_logs": True,
     }
     run_with_kwargs(kwargs)