nemo-evaluator-launcher 0.1.21__py3-none-any.whl → 0.1.23__py3-none-any.whl

nemo_evaluator_launcher/common/helpers.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 import base64
+import copy
 import datetime
 from dataclasses import dataclass
 from typing import Optional
@@ -170,7 +171,9 @@ def get_eval_factory_command(
         + "&& $cmd run_eval --run_config config_ef.yaml"
     )
 
-    overrides = merged_nemo_evaluator_config.get("overrides", {})
+    # NOTE: see note and test about deprecating that.
+    overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
+    overrides.update(dict(user_task_config.get("overrides", {})))
     # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
     # Consider passing a JSON object on Eval Factory side
     overrides = {

nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 21
+PATCH = 23
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

nemo_evaluator_launcher/resources/mapping.toml

@@ -226,7 +226,7 @@ required_env_vars = []
 [bigcode-evaluation-harness.tasks.completions.humaneval]
 required_env_vars = []
 
-[bigcode-evaluation-harness.tasks.completions.humaneval_instruct]
+[bigcode-evaluation-harness.tasks.chat.humaneval_instruct]
 
 
 ###############################################################################
@@ -304,29 +304,29 @@ required_env_vars = []
 
 ###############################################################################
 # NOTE(wprazuch): to verify if the tasks need any env var setting
-[nemo-skills]
+[nemo_skills]
 container = "nvcr.io/nvidia/eval-factory/nemo_skills:25.09.1"
 
-[nemo-skills.tasks.chat.ns_aime2024]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_aime2024]
+required_env_vars = ["JUDGE_API_KEY"]
 
-[nemo-skills.tasks.chat.ns_aime2025]
+[nemo_skills.tasks.chat.ns_aime2025]
 required_env_vars = []
 
-[nemo-skills.tasks.chat.ns_bfcl_v3]
+[nemo_skills.tasks.chat.ns_bfcl_v3]
 required_env_vars = []
 
-[nemo-skills.tasks.chat.ns_gpqa]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_gpqa]
+required_env_vars = ["HF_TOKEN"]
 
-[nemo-skills.tasks.chat.ns_hle]
+[nemo_skills.tasks.chat.ns_hle]
 required_env_vars = []
 
-[nemo-skills.tasks.chat.ns_mmlu]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_mmlu]
+required_env_vars = ["HF_TOKEN"]
 
-[nemo-skills.tasks.chat.ns_mmlu_pro]
-required_env_vars = []
+[nemo_skills.tasks.chat.ns_mmlu_pro]
+required_env_vars = ["HF_TOKEN"]
 
 ###############################################################################
 [safety-harness]

{nemo_evaluator_launcher-0.1.21.dist-info → nemo_evaluator_launcher-0.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.21
+Version: 0.1.23
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
 Provides-Extra: wandb
 Requires-Dist: wandb>=0.15.0; extra == "wandb"
 Provides-Extra: gsheets
-Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
+Requires-Dist: gspread>=5.0.0; extra == "gsheets"
 Provides-Extra: exporters
 Requires-Dist: mlflow; extra == "exporters"
 Requires-Dist: wandb; extra == "exporters"

{nemo_evaluator_launcher-0.1.21.dist-info → nemo_evaluator_launcher-0.1.23.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nemo_evaluator_launcher/__init__.py,sha256=GT38zGwbvBOSeU52WCRx-n9N49LvLGEV1PItgKC8orA,2320
-nemo_evaluator_launcher/package_info.py,sha256=oc2OZPiKTMsAYyASxBeE1qPWFlJvxPh7Ey9ajBjdEOk,1586
+nemo_evaluator_launcher/package_info.py,sha256=BXDKj0jZhJPCGvEBfg-lEvW1XIDijovEWP6SqRlqfdA,1586
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
 nemo_evaluator_launcher/api/functional.py,sha256=T1HTIeiTXb-APWP7lPPTwFam4vFOApZCScRi6tMp538,27648
 nemo_evaluator_launcher/api/types.py,sha256=W7ZQ9ZTPR6YxInxxsKE6NxuuQAg4pVYz6SRmFCFxY0A,3635
@@ -16,7 +16,7 @@ nemo_evaluator_launcher/cli/status.py,sha256=ANdu0JYnfKNvd1gXmdu_0FrbPG-g0A_R4le
 nemo_evaluator_launcher/cli/version.py,sha256=puMwIvkmfD3HESjftdTSP6T3Nc8J4cbz8uXWHJcTemY,2030
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=WPzg5Iu2ojvFpBuYahSt3voP_iEUpoO8NgqMLUBwFxA,9767
-nemo_evaluator_launcher/common/helpers.py,sha256=ohfTQoltPFG4lDbKbWR3KgcS7Nhlw2cTiEYml_aPhjA,11283
+nemo_evaluator_launcher/common/helpers.py,sha256=7O0cZYGndVaGYhu0npC0faspuD5NeBr2DfzwyAFvPSY,11422
 nemo_evaluator_launcher/common/logging_utils.py,sha256=LSMGeCyc_QwA6NyyNMKWgKNArz6HBlULEcZy33nlenQ,12214
 nemo_evaluator_launcher/common/mapping.py,sha256=tD3jWN7rm9-iJEFlENhYMt7adz8DKs67g3Xd43XIAMM,10731
 nemo_evaluator_launcher/common/printing_utils.py,sha256=YICPY-KhxjL5QNEFJNvYfnj6_ArkZURDdP1pizqY-yU,2368
@@ -51,10 +51,10 @@ nemo_evaluator_launcher/exporters/mlflow.py,sha256=qFEe9774s31Uybpah7kiz4BfZUDMy
 nemo_evaluator_launcher/exporters/registry.py,sha256=XsPTv_SBAFjcErO6BJ3OHqs3EvXQpLeyKRJuK9Ql4_M,1299
 nemo_evaluator_launcher/exporters/utils.py,sha256=XZVgTDmoa20tjEMwez0oUSpYpjt3ILV75D4KWuHtZ80,23119
 nemo_evaluator_launcher/exporters/wandb.py,sha256=1qRUV_YE1Ury7rH7KH65AabR7gmEQ38kXBh2XrfiEpE,18082
-nemo_evaluator_launcher/resources/mapping.toml,sha256=3Dgvx_C9M_TdxQ2nZn-QkNEYY7Y3MUMKhCMbBv8cIP4,12233
-nemo_evaluator_launcher-0.1.21.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.21.dist-info/METADATA,sha256=Nww4E5YEWFRG4tTVnrvAjLSXzPYxs27C5EDaL-y9W0U,28725
-nemo_evaluator_launcher-0.1.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.21.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
-nemo_evaluator_launcher-0.1.21.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.21.dist-info/RECORD,,
+nemo_evaluator_launcher/resources/mapping.toml,sha256=7M0LAFHI_EUAEPgpPBP_5HxugM0EUa8mUqg-dW4mKmE,12271
+nemo_evaluator_launcher-0.1.23.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.23.dist-info/METADATA,sha256=QupygFYxAQzCpOAAnGGVjl5QLvbQLqKNZE2VPmRvwAI,28725
+nemo_evaluator_launcher-0.1.23.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.23.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
+nemo_evaluator_launcher-0.1.23.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.23.dist-info/RECORD,,