nemo-evaluator-launcher 0.1.21__tar.gz → 0.1.23__tar.gz

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.21
+Version: 0.1.23
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
 Provides-Extra: wandb
 Requires-Dist: wandb>=0.15.0; extra == "wandb"
 Provides-Extra: gsheets
-Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
+Requires-Dist: gspread>=5.0.0; extra == "gsheets"
 Provides-Extra: exporters
 Requires-Dist: mlflow; extra == "exporters"
 Requires-Dist: wandb; extra == "exporters"

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/pyproject.toml

@@ -40,7 +40,7 @@ repository = "https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-l
 [project.optional-dependencies]
 mlflow = ["mlflow>=2.8.0"]
 wandb = ["wandb>=0.15.0"]
-gsheets = ["gsheets>=0.1.0"]
+gsheets = ["gspread>=5.0.0"]
 exporters = ["mlflow", "wandb", "gsheets"]
 all = ["mlflow", "wandb", "gsheets"]
 

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/helpers.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 import base64
+import copy
 import datetime
 from dataclasses import dataclass
 from typing import Optional
@@ -170,7 +171,9 @@ def get_eval_factory_command(
         + "&& $cmd run_eval --run_config config_ef.yaml"
     )
 
-    overrides = merged_nemo_evaluator_config.get("overrides", {})
+    # NOTE: see note and test about deprecating that.
+    overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
+    overrides.update(dict(user_task_config.get("overrides", {})))
     # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
     # Consider passing a JSON object on Eval Factory side
     overrides = {

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 21
+PATCH = 23
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/resources/mapping.toml

@@ -226,7 +226,7 @@ required_env_vars = []
 [bigcode-evaluation-harness.tasks.completions.humaneval]
 required_env_vars = []
 
-[bigcode-evaluation-harness.tasks.completions.humaneval_instruct]
+[bigcode-evaluation-harness.tasks.chat.humaneval_instruct]
 
 
 ###############################################################################
@@ -304,29 +304,29 @@ required_env_vars = []
 
 ###############################################################################
 # NOTE(wprazuch): to verify if the tasks need any env var setting
-[nemo-skills]
+[nemo_skills]
 container = "nvcr.io/nvidia/eval-factory/nemo_skills:25.09.1"
 
-[nemo-skills.tasks.chat.ns_aime2024]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_aime2024]
+required_env_vars = ["JUDGE_API_KEY"]
 
-[nemo-skills.tasks.chat.ns_aime2025]
+[nemo_skills.tasks.chat.ns_aime2025]
 required_env_vars = []
 
-[nemo-skills.tasks.chat.ns_bfcl_v3]
+[nemo_skills.tasks.chat.ns_bfcl_v3]
 required_env_vars = []
 
-[nemo-skills.tasks.chat.ns_gpqa]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_gpqa]
+required_env_vars = ["HF_TOKEN"]
 
-[nemo-skills.tasks.chat.ns_hle]
+[nemo_skills.tasks.chat.ns_hle]
 required_env_vars = []
 
-[nemo-skills.tasks.chat.ns_mmlu]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_mmlu]
+required_env_vars = ["HF_TOKEN"]
 
-[nemo-skills.tasks.chat.ns_mmlu_pro]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_mmlu_pro]
+required_env_vars = ["HF_TOKEN"]
 
 ###############################################################################
 [safety-harness]

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.21
+Version: 0.1.23
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
 Provides-Extra: wandb
 Requires-Dist: wandb>=0.15.0; extra == "wandb"
 Provides-Extra: gsheets
-Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
+Requires-Dist: gspread>=5.0.0; extra == "gsheets"
 Provides-Extra: exporters
 Requires-Dist: mlflow; extra == "exporters"
 Requires-Dist: wandb; extra == "exporters"

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/requires.txt

@@ -21,7 +21,7 @@ wandb
 gsheets
 
 [gsheets]
-gsheets>=0.1.0
+gspread>=5.0.0
 
 [mlflow]
 mlflow>=2.8.0