nemo-evaluator-launcher 0.1.19__py3-none-any.whl → 0.1.41__py3-none-any.whl

nemo_evaluator_launcher/executors/slurm/proxy.cfg.template

@@ -0,0 +1,26 @@
+global
+    log stdout format raw local0
+    maxconn 4096
+
+defaults
+    log     global
+    mode    http
+    option  httplog
+    timeout connect 10s
+    timeout client  100000s
+    timeout server  100000s
+
+frontend service_frontend
+    bind *:{{ haproxy_port }}
+    default_backend service_backend
+
+backend service_backend
+    mode http
+    option httpchk GET {{ health_check_path }}
+    http-check expect status {{ health_check_status }}
+    option http-server-close
+    balance leastconn
+{% for node in nodes %}
+    server node{{ loop.index }} {{ node.ip }}:{{ node.port }} check
+{% endfor %}
+

nemo_evaluator_launcher/exporters/utils.py

@@ -471,15 +471,12 @@ def _extract_metrics_from_results(results: dict) -> Dict[str, float]:
         section_data = results.get(section)
         if isinstance(section_data, dict):
             for task_name, task_data in section_data.items():
-                if isinstance(task_data, dict) and "metrics" in task_data:
-                    task_metrics = _extract_task_metrics(
-                        task_name, task_data["metrics"]
-                    )
-                    _safe_update_metrics(
-                        target=metrics,
-                        source=task_metrics,
-                        context=f" while extracting results for task '{task_name}'",
-                    )
+                task_metrics = _extract_task_metrics(task_name, task_data)
+                _safe_update_metrics(
+                    target=metrics,
+                    source=task_metrics,
+                    context=f" while extracting results for task '{task_name}'",
+                )
     return metrics
 
 
@@ -518,54 +515,43 @@ def _extract_from_json_files(artifacts_dir: Path) -> Dict[str, float]:
     return metrics
 
 
-def _extract_task_metrics(task_name: str, metrics_data: dict) -> Dict[str, float]:
+def _extract_task_metrics(task_name: str, task_data: dict) -> Dict[str, float]:
     """Extract metrics from a task's metrics data."""
     extracted = {}
-    score_patterns = [
-        "acc",
-        "accuracy",
-        "score",
-        "exact_match",
-        "f1",
-        "em",
-        "pass@1",
-        "pass@k",
-    ]
 
-    for metric_name, metric_data in metrics_data.items():
-        # Only extract score-like metrics
-        if not any(pattern in metric_name.lower() for pattern in score_patterns):
-            continue
+    metrics_data = task_data.get("metrics", {})
+    if "groups" in task_data:
+        for group_name, group_data in task_data["groups"].items():
+            group_extracted = _extract_task_metrics(
+                f"{task_name}_{group_name}", group_data
+            )
+            _safe_update_metrics(
+                target=extracted,
+                source=group_extracted,
+                context=f" in task '{task_name}'",
+            )
 
+    for metric_name, metric_data in metrics_data.items():
         try:
-            if isinstance(metric_data, dict):
-                if "scores" in metric_data:
-                    # Handle nested scores (e.g., mmlu macro/micro)
-                    for score_type, score_data in metric_data["scores"].items():
-                        if isinstance(score_data, dict) and "value" in score_data:
-                            key = f"{task_name}_{metric_name}_{score_type}"
-                            _safe_set_metric(
-                                container=extracted,
-                                key=key,
-                                new_value=score_data["value"],
-                                context=f" in task '{task_name}'",
-                            )
-                elif "value" in metric_data:
+            for score_type, score_data in metric_data["scores"].items():
+                if score_type != metric_name:
+                    key = f"{task_name}_{metric_name}_{score_type}"
+                else:
                     key = f"{task_name}_{metric_name}"
-                    _safe_set_metric(
-                        container=extracted,
-                        key=key,
-                        new_value=metric_data["value"],
-                        context=f" in task '{task_name}'",
-                    )
-            elif isinstance(metric_data, (int, float)):
-                key = f"{task_name}_{metric_name}"
                 _safe_set_metric(
                     container=extracted,
                     key=key,
-                    new_value=metric_data,
+                    new_value=score_data["value"],
                     context=f" in task '{task_name}'",
                 )
+                for stat_name, stat_value in metric_data.get("stats", {}).items():
+                    stats_key = f"{key}_{stat_name}"
+                    _safe_set_metric(
+                        container=extracted,
+                        key=stats_key,
+                        new_value=stat_value,
+                        context=f" in task '{task_name}'",
+                    )
         except (ValueError, TypeError) as e:
             logger.warning(
                 f"Failed to extract metric {metric_name} for task {task_name}: {e}"

nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 19
+PATCH = 41
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

nemo_evaluator_launcher/resources/mapping.toml

@@ -1,6 +1,6 @@
 # NOTE(agronskiy): checked parity
 [lm-evaluation-harness]
-container = "nvcr.io/nvidia/eval-factory/lm-evaluation-harness:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/lm-evaluation-harness:25.10"
 
 [lm-evaluation-harness.tasks.chat.ifeval]
 required_env_vars = []
@@ -79,6 +79,8 @@ required_env_vars = []
 
 [lm-evaluation-harness.tasks.chat.mmlu_redux_instruct]
 
+[lm-evaluation-harness.tasks.chat.mmlu_cot_0_shot_chat]
+
 [lm-evaluation-harness.tasks.completions.gsm8k]
 required_env_vars = []
 
@@ -124,7 +126,7 @@ required_env_vars = []
 ###############################################################################
 # NOTE(agronskiy): checked parity
 [mtbench]
-container = "nvcr.io/nvidia/eval-factory/mtbench:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/mtbench:25.10"
 
 [mtbench.tasks.chat.mtbench]
 
@@ -134,7 +136,7 @@ container = "nvcr.io/nvidia/eval-factory/mtbench:25.08.1"
 ###############################################################################
 # NOTE(agronskiy): checked parity
 [ifbench]
-container = "nvcr.io/nvidia/eval-factory/ifbench:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/ifbench:25.10"
 
 [ifbench.tasks.chat.ifbench]
 required_env_vars = []
@@ -142,7 +144,7 @@ required_env_vars = []
 
 ###############################################################################
 [simple_evals]
-container = "nvcr.io/nvidia/eval-factory/simple-evals:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/simple-evals:25.10"
 
 [simple_evals.tasks.chat.gpqa_diamond]
 required_env_vars = ["HF_TOKEN"]
@@ -213,7 +215,7 @@ required_env_vars = []
 ###############################################################################
 # NOTE(agronskiy): checked parity
 [bigcode-evaluation-harness]
-container = "nvcr.io/nvidia/eval-factory/bigcode-evaluation-harness:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/bigcode-evaluation-harness:25.10"
 
 [bigcode-evaluation-harness.tasks.chat.mbpp]
 required_env_vars = []
@@ -226,12 +228,12 @@ required_env_vars = []
 [bigcode-evaluation-harness.tasks.completions.humaneval]
 required_env_vars = []
 
-[bigcode-evaluation-harness.tasks.completions.humaneval_instruct]
+[bigcode-evaluation-harness.tasks.chat.humaneval_instruct]
 
 
 ###############################################################################
 [livecodebench]
-container = "nvcr.io/nvidia/eval-factory/livecodebench:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/livecodebench:25.10"
 
 [livecodebench.tasks.chat.livecodebench_0724_0125]
 required_env_vars = []
@@ -242,7 +244,7 @@ required_env_vars = []
 
 ###############################################################################
 [scicode]
-container = "nvcr.io/nvidia/eval-factory/scicode:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/scicode:25.10"
 
 [scicode.tasks.chat.aa_scicode]
 required_env_vars = []
@@ -250,7 +252,7 @@ required_env_vars = []
 
 ###############################################################################
 [hle]
-container = "nvcr.io/nvidia/eval-factory/hle:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/hle:25.10"
 
 [hle.tasks.chat.hle]
 required_env_vars = ["HF_TOKEN", "OPENAI_CLIENT_ID", "OPENAI_CLIENT_SECRET"]
@@ -258,7 +260,7 @@ required_env_vars = ["HF_TOKEN", "OPENAI_CLIENT_ID", "OPENAI_CLIENT_SECRET"]
 
 ###############################################################################
 [bfcl]
-container = "nvcr.io/nvidia/eval-factory/bfcl:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/bfcl:25.10"
 
 [bfcl.tasks.chat.bfclv2_ast_prompting]
 required_env_vars = []
@@ -267,9 +269,20 @@ required_env_vars = []
 required_env_vars = []
 
 
+###############################################################################
+[profbench]
+container = "nvcr.io/nvidia/eval-factory/profbench:25.10"
+
+[profbench.tasks.chat.llm_judge]
+required_env_vars = []
+
+[profbench.tasks.chat.report_generation]
+required_env_vars = []
+
+
 ###############################################################################
 [vlmevalkit]
-container = "nvcr.io/nvidia/eval-factory/vlmevalkit:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/vlmevalkit:25.10"
 
 [vlmevalkit.tasks.vlm.ocrbench]
 required_env_vars = []
@@ -286,15 +299,43 @@ required_env_vars = ["OPENAI_CLIENT_ID", "OPENAI_CLIENT_SECRET"]
 
 ###############################################################################
 [garak]
-container = "nvcr.io/nvidia/eval-factory/garak:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/garak:25.10"
 
 [garak.tasks.chat.garak]
 required_env_vars = []
 
+###############################################################################
+# NOTE(wprazuch): to verify if the tasks need any env var setting
+[nemo_skills]
+container = "nvcr.io/nvidia/eval-factory/nemo_skills:25.10"
+
+[nemo_skills.tasks.chat.ns_aime2024]
+required_env_vars = ["JUDGE_API_KEY"]
+
+[nemo_skills.tasks.chat.ns_aime2025]
+required_env_vars = []
+
+[nemo_skills.tasks.chat.ns_bfcl_v3]
+required_env_vars = []
+
+[nemo_skills.tasks.chat.ns_gpqa]
+required_env_vars = ["HF_TOKEN"]
+
+[nemo_skills.tasks.chat.ns_hle]
+required_env_vars = []
+
+[nemo_skills.tasks.chat.ns_mmlu]
+required_env_vars = ["HF_TOKEN"]
+
+[nemo_skills.tasks.chat.ns_mmlu_pro]
+required_env_vars = ["HF_TOKEN"]
+
+[nemo_skills.tasks.chat.ns_aa_lcr]
+required_env_vars = ["JUDGE_API_KEY"]
 
 ###############################################################################
 [safety-harness]
-container = "nvcr.io/nvidia/eval-factory/safety-harness:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/safety-harness:25.10"
 
 [safety-harness.tasks.chat.aegis_v2]
 required_env_vars = ["HF_TOKEN"]
@@ -303,7 +344,7 @@ required_env_vars = ["HF_TOKEN"]
 ###############################################################################
 # NOTE(agronskiy): checked parity
 [helm]
-container = "nvcr.io/nvidia/eval-factory/helm:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/helm:25.10"
 
 [helm.tasks.chat.medcalc_bench]
 
@@ -339,6 +380,6 @@ container = "nvcr.io/nvidia/eval-factory/helm:25.08.1"
 ###############################################################################
 # NOTE(agronskiy): checked parity
 [tooltalk]
-container = "nvcr.io/nvidia/eval-factory/tooltalk:25.08.1"
+container = "nvcr.io/nvidia/eval-factory/tooltalk:25.10"
 
 [tooltalk.tasks.chat.tooltalk]

{nemo_evaluator_launcher-0.1.19.dist-info → nemo_evaluator_launcher-0.1.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.19
+Version: 0.1.41
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -458,7 +458,7 @@ License:
               OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
               SOFTWARE.
         
-Project-URL: homepage, https://github.com/NVIDIA-NeMo/Eval
+Project-URL: homepage, https://github.com/NVIDIA-NeMo/Evaluator
 Project-URL: repository, https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-launcher
 Keywords: deep learning,evaluations,machine learning,gpu,NLP,pytorch,torch
 Requires-Python: <3.14,>=3.10
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
 Provides-Extra: wandb
 Requires-Dist: wandb>=0.15.0; extra == "wandb"
 Provides-Extra: gsheets
-Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
+Requires-Dist: gspread>=5.0.0; extra == "gsheets"
 Provides-Extra: exporters
 Requires-Dist: mlflow; extra == "exporters"
 Requires-Dist: wandb; extra == "exporters"

{nemo_evaluator_launcher-0.1.19.dist-info → nemo_evaluator_launcher-0.1.41.dist-info}/RECORD

@@ -1,60 +1,62 @@
 nemo_evaluator_launcher/__init__.py,sha256=GT38zGwbvBOSeU52WCRx-n9N49LvLGEV1PItgKC8orA,2320
-nemo_evaluator_launcher/package_info.py,sha256=GpwtisWr161CuNwMc4fqUv_0wC9cvJelVoIoC6M6pjY,1586
+nemo_evaluator_launcher/package_info.py,sha256=RGnruVj93NUhGw5fqI4DqujBesAV9Ixf05LquZO-JBU,1586
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
-nemo_evaluator_launcher/api/functional.py,sha256=T1HTIeiTXb-APWP7lPPTwFam4vFOApZCScRi6tMp538,27648
+nemo_evaluator_launcher/api/functional.py,sha256=4Se7avg6Wde4EK7is3y-KjXV5qR9RGBpe8zjwjej4M8,31859
 nemo_evaluator_launcher/api/types.py,sha256=W7ZQ9ZTPR6YxInxxsKE6NxuuQAg4pVYz6SRmFCFxY0A,3635
 nemo_evaluator_launcher/api/utils.py,sha256=q5HArRj7PKgBfeH3bOX8q1U97yMyQQp72yRRA5JP9PE,818
 nemo_evaluator_launcher/cli/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/cli/export.py,sha256=GRXxusKDq_1qjMKN6MKOIjZ8x4u5ERgXwHSAGrvsGCY,11211
 nemo_evaluator_launcher/cli/info.py,sha256=2dZA2BqXpTG1wO_Wzt6Ol9ZNJzJJ0PibOB0hLFZpL14,20705
 nemo_evaluator_launcher/cli/kill.py,sha256=C-4PWmMu8mIITo92o5AHxtq_s-8Cckbp7wAlG0I_ylw,1323
+nemo_evaluator_launcher/cli/logs.py,sha256=eYgfDbHNZhw1xo56bwCIR1sGQYJIPPzzUJu-pqseDWk,3774
 nemo_evaluator_launcher/cli/ls_runs.py,sha256=vJTwRdhVKLolnJuP8AnnQdJBE-BKZfCcCypLKSz5gqs,4942
 nemo_evaluator_launcher/cli/ls_tasks.py,sha256=Pd2lBQOQBNHBWrjk4tZg0SQ9Ul9F2Ak-zOyh-G9x-DY,5293
-nemo_evaluator_launcher/cli/main.py,sha256=MQo-DcVF_f0aBgnVUic5PP1qlfXXs0Evkd1X4EXdOvA,7217
-nemo_evaluator_launcher/cli/run.py,sha256=3-C_GvaIg9IxqpvC4P3h3lHcHdjB94Zpgq0ccOXVcpw,7503
+nemo_evaluator_launcher/cli/main.py,sha256=iNEAeE7_dfDia4oCjUYhYikJJJXg9cMlbGADwMfMtfM,7613
+nemo_evaluator_launcher/cli/run.py,sha256=XKF4nRdijhFH01JvMtWXav6tc6Xp7tAhbSnk32q8Ma4,9966
 nemo_evaluator_launcher/cli/status.py,sha256=ANdu0JYnfKNvd1gXmdu_0FrbPG-g0A_R4leOuNXzenQ,5947
-nemo_evaluator_launcher/cli/version.py,sha256=puMwIvkmfD3HESjftdTSP6T3Nc8J4cbz8uXWHJcTemY,2030
+nemo_evaluator_launcher/cli/version.py,sha256=GgMNTAd4S0bu3t-uVfVedAf7p6pymWDDwOaNm4WHOxQ,1998
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=WPzg5Iu2ojvFpBuYahSt3voP_iEUpoO8NgqMLUBwFxA,9767
-nemo_evaluator_launcher/common/helpers.py,sha256=T51kDqwtvUStM43FVUJdX9m6ja7cwK0sKX9-7vd8zXE,9327
-nemo_evaluator_launcher/common/logging_utils.py,sha256=8UMAQ22t5NAJRDZtI0gVbdKUlNAiG23WQwZZ0HwzOT4,11843
+nemo_evaluator_launcher/common/helpers.py,sha256=MrLHPlOSn8GGc7sxB0xlDmUN0uWluSgVmDLUBwuo7pU,13540
+nemo_evaluator_launcher/common/logging_utils.py,sha256=7QkWlpA80QN5ipTUFJ198IiAsPRS36C6ISAAtNverbA,12338
 nemo_evaluator_launcher/common/mapping.py,sha256=tD3jWN7rm9-iJEFlENhYMt7adz8DKs67g3Xd43XIAMM,10731
-nemo_evaluator_launcher/common/printing_utils.py,sha256=YICPY-KhxjL5QNEFJNvYfnj6_ArkZURDdP1pizqY-yU,2368
+nemo_evaluator_launcher/common/printing_utils.py,sha256=QPjLqFlp0dd18RY__UYo5-yG8OqABaa78cpzMhZVCeo,2579
 nemo_evaluator_launcher/configs/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/configs/default.yaml,sha256=JHFjSl3KByhggRMTTo9nesQATVoz18PJIV6KM5Wng64,974
 nemo_evaluator_launcher/configs/deployment/generic.yaml,sha256=8_Z0fcjZuH6GfV9jJkY_8CS18Tbsn0gV-nK1LfGr_Vg,1262
 nemo_evaluator_launcher/configs/deployment/nim.yaml,sha256=hRJGwCR2XIS3bUFWkXzmroLep81KaHa3Sn2_edfWbkU,1266
 nemo_evaluator_launcher/configs/deployment/none.yaml,sha256=buPada1yMz9ClPDbq63vPDzLGr_IubTLia91DG3i5Lo,684
-nemo_evaluator_launcher/configs/deployment/sglang.yaml,sha256=2CrnmpdEr5HvUbZW6k_wFPK41q6e4WoS7CJKAcgKlck,1273
-nemo_evaluator_launcher/configs/deployment/trtllm.yaml,sha256=nZF1ueCF9uMticElgePPy8ecq2vbfvWxj_5YwQTJ--k,570
-nemo_evaluator_launcher/configs/deployment/vllm.yaml,sha256=NOAfK7hALTeVxjNJ63zEkb0vnefeRuAQCh6OZwe18pA,1476
+nemo_evaluator_launcher/configs/deployment/sglang.yaml,sha256=Yy2uOko7-HeJ1fE0LHK3AHUOAzaeVSfnDp6f7fRBA9s,1358
+nemo_evaluator_launcher/configs/deployment/trtllm.yaml,sha256=1-001_ylJOiQQiqhcIkX_8SFTkOGx47odB_4Myug_9A,568
+nemo_evaluator_launcher/configs/deployment/vllm.yaml,sha256=5ORXOxrkMospl_iLUMTuOV0o4EOVO5VBEb5jAldCB_s,1458
 nemo_evaluator_launcher/configs/execution/local.yaml,sha256=0XtVXHeCK-zAoTURxnf4pr9RCe46HUCpKVzcfPlskz4,740
 nemo_evaluator_launcher/configs/execution/lepton/default.yaml,sha256=SRSMxohEtafzb-QS_oz4kP-RBgigRCAZgYwKkXtjymY,2930
-nemo_evaluator_launcher/configs/execution/slurm/default.yaml,sha256=zTevKSKch7yl4D0wGh4BjDBuI0YAiJSljVKwD7G8Cww,1328
+nemo_evaluator_launcher/configs/execution/slurm/default.yaml,sha256=noQO06oEd7INQAZBw_lH7OHKQteDH4COk5-4I5I4Tx0,1733
 nemo_evaluator_launcher/executors/__init__.py,sha256=mSU1op5r7R_vqOCLDP84z6utfFgXOIl_1vBzN7KOC6o,1042
-nemo_evaluator_launcher/executors/base.py,sha256=4BOz2-jMG1OJ-5o5qCh-SJqLUE64YJWlnmB9hc1p4Pc,4040
+nemo_evaluator_launcher/executors/base.py,sha256=QC1HDyTqXJFQMPMz48A-DLOFu3le4NFvVFzvPyxDFfE,5337
 nemo_evaluator_launcher/executors/registry.py,sha256=8QXSrsJyHeNi8iSttJ8KWQLXmZve1vxnnCNw_CkeopI,1409
 nemo_evaluator_launcher/executors/lepton/__init__.py,sha256=F_7yuBaYQ6WWTcptADdkL3AIZ_jXJQHGgKag-Hm7BbQ,698
-nemo_evaluator_launcher/executors/lepton/deployment_helpers.py,sha256=AAIlHHn-WifevNosug0DlSDLN6NtjkclEu5LHyu1xq8,21799
-nemo_evaluator_launcher/executors/lepton/executor.py,sha256=BeKZFflrooh_gbGoujY-cKOkarS-_VI0AoER91t5zvA,40240
+nemo_evaluator_launcher/executors/lepton/deployment_helpers.py,sha256=vzAWaNmacRsloURwVIc6LsP-xpKwvr7vsPlFIFTFdLQ,23234
+nemo_evaluator_launcher/executors/lepton/executor.py,sha256=gKls0DNTVyFVOh3rjIPAVruaAeQ5qmVcNoZctlLnb5w,44347
 nemo_evaluator_launcher/executors/lepton/job_helpers.py,sha256=6baTxcygfP1oFgAJ7I9EL4xRlcJDWqbqzZoE1CRrwSk,13528
 nemo_evaluator_launcher/executors/local/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/local/executor.py,sha256=OUJ4B8qvE_mz9tXTYkwDqEjtOLtJzzbvOHDW35LYR4U,21431
-nemo_evaluator_launcher/executors/local/run.template.sh,sha256=uPofo2g8f7LsS0uL0YK0Y5YFFCQ5fAeiAkYMGBTalpg,3966
+nemo_evaluator_launcher/executors/local/executor.py,sha256=ElRNKv8GVlcOHP23Uc67RiSleI6GfWpBK0e8w6VcXRk,36880
+nemo_evaluator_launcher/executors/local/run.template.sh,sha256=S6qLqnqA3B4P_-ngklCU5dO9rKV0Qom5NWDwDHf5i0g,6103
 nemo_evaluator_launcher/executors/slurm/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/slurm/executor.py,sha256=J_0sYzCcbcGzitf_oC9ncguoD2eG-K3osJxmZTW-9tE,40096
+nemo_evaluator_launcher/executors/slurm/executor.py,sha256=2miKvU5kHr5fs706MxgAjDPE8nuqmOuWjnzni2F97qo,55108
+nemo_evaluator_launcher/executors/slurm/proxy.cfg.template,sha256=nPg_JyBHPcjI1RSxDvl-lBgcbakrbZLl4IAXNBc8LM0,594
 nemo_evaluator_launcher/exporters/__init__.py,sha256=mBXG9FG48FeYrs8sF0zA2mgo1eqBmRgoml7zjJrqDso,1323
 nemo_evaluator_launcher/exporters/base.py,sha256=0BEqS-Zjez-KsrGE9yfo8S5w2uwMW3btBZve3SiiUp0,4307
 nemo_evaluator_launcher/exporters/gsheets.py,sha256=hBOL3vaomCW2fPMDEOQWkZkFCgF4jCoS4U5ZlsNVENs,15911
 nemo_evaluator_launcher/exporters/local.py,sha256=pnC_FhQ1PRYUllN0BKGVq8RfuPoMMEv_EaCf5eTQ8OI,19766
 nemo_evaluator_launcher/exporters/mlflow.py,sha256=qFEe9774s31Uybpah7kiz4BfZUDMyjo74DYuL7d7zNE,24727
 nemo_evaluator_launcher/exporters/registry.py,sha256=XsPTv_SBAFjcErO6BJ3OHqs3EvXQpLeyKRJuK9Ql4_M,1299
-nemo_evaluator_launcher/exporters/utils.py,sha256=XZVgTDmoa20tjEMwez0oUSpYpjt3ILV75D4KWuHtZ80,23119
+nemo_evaluator_launcher/exporters/utils.py,sha256=5HGBKoFTSCng-GhAJpvfAa7N52_r9UDyrGlBLqRiiGo,22587
 nemo_evaluator_launcher/exporters/wandb.py,sha256=1qRUV_YE1Ury7rH7KH65AabR7gmEQ38kXBh2XrfiEpE,18082
-nemo_evaluator_launcher/resources/mapping.toml,sha256=uOg4Y-gDXXskbbba2vuwJ5FLJ3W0kSZz7Fap_nJnFQc,11322
-nemo_evaluator_launcher-0.1.19.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.19.dist-info/METADATA,sha256=4_GKlbu2rptZcvCu98Z4diI7eC3YHwyKlC3ByXOEKnk,28725
-nemo_evaluator_launcher-0.1.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.19.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
-nemo_evaluator_launcher-0.1.19.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.19.dist-info/RECORD,,
+nemo_evaluator_launcher/resources/mapping.toml,sha256=oUeYpfOTyh5iiQs1rdc9zo-j_1P8ReWAzole2wYDFgg,12370
+nemo_evaluator_launcher-0.1.41.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.41.dist-info/METADATA,sha256=rCHDM0bplzE1q9gmi1GYzG1oQBpia3NfToSOEmN-JiU,28730
+nemo_evaluator_launcher-0.1.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.41.dist-info/entry_points.txt,sha256=HPmLybw-y1y4NxjK7slFS4mtTB7p4pnjRvCtJLvSiZs,174
+nemo_evaluator_launcher-0.1.41.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.41.dist-info/RECORD,,

{nemo_evaluator_launcher-0.1.19.dist-info → nemo_evaluator_launcher-0.1.41.dist-info}/entry_points.txt

@@ -1,3 +1,4 @@
 [console_scripts]
+nel = nemo_evaluator_launcher.cli.main:main
 nemo-evaluator-launcher = nemo_evaluator_launcher.cli.main:main
 nv-eval = nemo_evaluator_launcher.cli.main:main