nemo-evaluator-launcher 0.1.41__py3-none-any.whl → 0.1.56__py3-none-any.whl

nemo_evaluator_launcher/resources/mapping.toml

@@ -1,385 +1,93 @@
-# NOTE(agronskiy): checked parity
-[lm-evaluation-harness]
-container = "nvcr.io/nvidia/eval-factory/lm-evaluation-harness:25.10"
-
-[lm-evaluation-harness.tasks.chat.ifeval]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.chat.mmlu_prox]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.completions.mmlu]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.completions.mmlu_pro]
-
-[lm-evaluation-harness.tasks.completions.global_mmlu]
-[lm-evaluation-harness.tasks.completions.global_mmlu_ar]
-[lm-evaluation-harness.tasks.completions.global_mmlu_bn]
-[lm-evaluation-harness.tasks.completions.global_mmlu_de]
-[lm-evaluation-harness.tasks.completions.global_mmlu_en]
-[lm-evaluation-harness.tasks.completions.global_mmlu_es]
-[lm-evaluation-harness.tasks.completions.global_mmlu_fr]
-[lm-evaluation-harness.tasks.completions.global_mmlu_hi]
-[lm-evaluation-harness.tasks.completions.global_mmlu_id]
-[lm-evaluation-harness.tasks.completions.global_mmlu_it]
-[lm-evaluation-harness.tasks.completions.global_mmlu_ja]
-[lm-evaluation-harness.tasks.completions.global_mmlu_ko]
-[lm-evaluation-harness.tasks.completions.global_mmlu_pt]
-[lm-evaluation-harness.tasks.completions.global_mmlu_sw]
-[lm-evaluation-harness.tasks.completions.global_mmlu_yo]
-[lm-evaluation-harness.tasks.completions.global_mmlu_zh]
-
-[lm-evaluation-harness.tasks.completions.global_mmlu_full]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_am]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ar]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_bn]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_cs]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_de]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_el]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_en]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_es]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_fa]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_fil]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_fr]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ha]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_he]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_hi]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_id]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ig]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_it]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ja]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ko]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ky]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_lt]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_mg]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ms]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ne]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_nl]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ny]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_pl]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_pt]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ro]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_ru]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_si]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_sn]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_so]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_sr]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_sv]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_sw]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_te]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_tr]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_uk]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_vi]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_yo]
-[lm-evaluation-harness.tasks.completions.global_mmlu_full_zh]
-[lm-evaluation-harness.tasks.completions.mmlu_logits]
-
-[lm-evaluation-harness.tasks.chat.mmlu_instruct]
-
-[lm-evaluation-harness.tasks.chat.mmlu_redux_instruct]
-
-[lm-evaluation-harness.tasks.chat.mmlu_cot_0_shot_chat]
-
-[lm-evaluation-harness.tasks.completions.gsm8k]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.chat.gsm8k_cot_instruct]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.chat.gsm8k_cot_llama]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.chat.mgsm_cot]
-
-[lm-evaluation-harness.tasks.chat.gpqa_diamond_cot]
-
-[lm-evaluation-harness.tasks.completions.winogrande]
-
-[lm-evaluation-harness.tasks.completions.hellaswag]
-[lm-evaluation-harness.tasks.completions.hellaswag_multilingual]
-
-[lm-evaluation-harness.tasks.completions.commonsense_qa]
-
-[lm-evaluation-harness.tasks.completions.openbookqa]
-
-[lm-evaluation-harness.tasks.completions.piqa]
-
-[lm-evaluation-harness.tasks.completions.adlr_race]
-
-[lm-evaluation-harness.tasks.completions.social_iqa]
-
-[lm-evaluation-harness.tasks.completions.adlr_truthfulqa_mc2]
-[lm-evaluation-harness.tasks.completions.adlr_minerva_math_nemo]
-[lm-evaluation-harness.tasks.completions.adlr_arc_challenge_llama]
-[lm-evaluation-harness.tasks.completions.adlr_mmlu_pro_5_shot_base]
-[lm-evaluation-harness.tasks.completions.adlr_mbpp_sanitized_3shot_greedy]
-[lm-evaluation-harness.tasks.completions.adlr_mbppplus_greedy_sanitized]
-[lm-evaluation-harness.tasks.completions.adlr_humaneval_greedy]
-[lm-evaluation-harness.tasks.completions.adlr_humanevalplus_greedy]
-[lm-evaluation-harness.tasks.chat.adlr_gsm8k_fewshot_cot]
-required_env_vars = []
-
-[lm-evaluation-harness.tasks.completions.arc_multilingual]
+# Workflow diagram:
+# mapping.toml
+# (container = "....")
+# (# container-digest:sha256:....)              <---- CI: check digests are relevant
+#
+#    |
+#    |
+#    v
+# scripts/container_metadata_controller.py
+# (updates the toml
+#  AND
+# creates the resources/all_tasks_irs,
+# records TOML checksum)                        <---- pre-commit guard: checks TOML checksum
+#    |          \
+#    |           \
+#    |            ------------------->    make docs-build
+#    |                                  (builds docs on the fly)
+#    v
+# scripts/container_metadata_controller.py
+# (updates README, records checksum)           <----- pre-commit guard: checks TOML checksum
+#
 
+[lm-evaluation-harness]
+container = "nvcr.io/nvidia/eval-factory/lm-evaluation-harness:25.11"
+# container-digest:sha256:4405ea93535f0d5067313b4f78fabead639be31fa3b9b700d72150872e2b78c9
 
-###############################################################################
-# NOTE(agronskiy): checked parity
 [mtbench]
-container = "nvcr.io/nvidia/eval-factory/mtbench:25.10"
+container = "nvcr.io/nvidia/eval-factory/mtbench:25.11"
+# container-digest:sha256:285561e1d52d61876ab9b4620f9ed033460bf44458ca692df4467bf2c325d236
 
-[mtbench.tasks.chat.mtbench]
-
-[mtbench.tasks.chat.mtbench-cor1]
-
-
-###############################################################################
-# NOTE(agronskiy): checked parity
 [ifbench]
-container = "nvcr.io/nvidia/eval-factory/ifbench:25.10"
-
-[ifbench.tasks.chat.ifbench]
-required_env_vars = []
+container = "nvcr.io/nvidia/eval-factory/ifbench:25.11"
+# container-digest:sha256:6263f213cce6cbc7468260b53f308ee77a9aa8e39652d51903a96668ea4badf8
 
-
-###############################################################################
 [simple_evals]
-container = "nvcr.io/nvidia/eval-factory/simple-evals:25.10"
-
-[simple_evals.tasks.chat.gpqa_diamond]
-required_env_vars = ["HF_TOKEN"]
-
-[simple_evals.tasks.chat.gpqa_diamond_aa_v2]
-required_env_vars = ["HF_TOKEN"]
-
-[simple_evals.tasks.chat.gpqa_diamond_aa_v2_llama_4]
-required_env_vars = ["HF_TOKEN"]
-
-[simple_evals.tasks.chat.gpqa_diamond_nemo]
-required_env_vars = ["HF_TOKEN"]
-
-[simple_evals.tasks.chat.AA_math_test_500]
-required_env_vars = ["JUDGE_API_KEY"]
-
-[simple_evals.tasks.chat.math_test_500_nemo]
-required_env_vars = []
-
-[simple_evals.tasks.chat.aime_2024_nemo]
-required_env_vars = []
-
-[simple_evals.tasks.chat.AA_AIME_2024]
-required_env_vars = ["JUDGE_API_KEY"]
+container = "nvcr.io/nvidia/eval-factory/simple-evals:25.11"
+# container-digest:sha256:6e0ffeaa93376852112fedc777800676fb4ae69260563e99b5a85dff7846d6f3
 
-[simple_evals.tasks.chat.aime_2025_nemo]
-required_env_vars = []
-
-[simple_evals.tasks.chat.AIME_2025]
-required_env_vars = ["JUDGE_API_KEY"]
-
-[simple_evals.tasks.chat.humaneval]
-required_env_vars = []
-
-[simple_evals.tasks.chat.mgsm]
-required_env_vars = []
-
-[simple_evals.tasks.chat.mmlu_pro]
-required_env_vars = []
-
-[simple_evals.tasks.chat.mmlu]
-required_env_vars = []
-
-[simple_evals.tasks.chat.mmlu_llama_4]
-required_env_vars = []
-
-[simple_evals.tasks.chat.mmlu_pro_llama_4]
-required_env_vars = []
-
-[simple_evals.tasks.chat.mmlu_ar-lite]
-[simple_evals.tasks.chat.mmlu_bn-lite]
-[simple_evals.tasks.chat.mmlu_de-lite]
-[simple_evals.tasks.chat.mmlu_en-lite]
-[simple_evals.tasks.chat.mmlu_es-lite]
-[simple_evals.tasks.chat.mmlu_fr-lite]
-[simple_evals.tasks.chat.mmlu_hi-lite]
-[simple_evals.tasks.chat.mmlu_id-lite]
-[simple_evals.tasks.chat.mmlu_it-lite]
-[simple_evals.tasks.chat.mmlu_ja-lite]
-[simple_evals.tasks.chat.mmlu_ko-lite]
-[simple_evals.tasks.chat.mmlu_my-lite]
-[simple_evals.tasks.chat.mmlu_pt-lite]
-[simple_evals.tasks.chat.mmlu_sw-lite]
-[simple_evals.tasks.chat.mmlu_yo-lite]
-[simple_evals.tasks.chat.mmlu_zh-lite]
-
-
-###############################################################################
-# NOTE(agronskiy): checked parity
 [bigcode-evaluation-harness]
-container = "nvcr.io/nvidia/eval-factory/bigcode-evaluation-harness:25.10"
-
-[bigcode-evaluation-harness.tasks.chat.mbpp]
-required_env_vars = []
-
-[bigcode-evaluation-harness.tasks.chat.mbppplus]
+container = "nvcr.io/nvidia/eval-factory/bigcode-evaluation-harness:25.11"
+# container-digest:sha256:8263687c042abd46fb23dab8c3aa831bd048497cdb4bde196e1e9ecf9d06cf0b
 
-[bigcode-evaluation-harness.tasks.chat.mbppplus_nemo]
-required_env_vars = []
-
-[bigcode-evaluation-harness.tasks.completions.humaneval]
-required_env_vars = []
-
-[bigcode-evaluation-harness.tasks.chat.humaneval_instruct]
-
-
-###############################################################################
 [livecodebench]
-container = "nvcr.io/nvidia/eval-factory/livecodebench:25.10"
-
-[livecodebench.tasks.chat.livecodebench_0724_0125]
-required_env_vars = []
-
-[livecodebench.tasks.chat.livecodebench_0824_0225]
-required_env_vars = []
-
+container = "nvcr.io/nvidia/eval-factory/livecodebench:25.11"
+# container-digest:sha256:0ce84b69b79fb2815dfe2d8487efbc098a9e131662cad0fd3d61040d0354d571
 
-###############################################################################
 [scicode]
-container = "nvcr.io/nvidia/eval-factory/scicode:25.10"
+container = "nvcr.io/nvidia/eval-factory/scicode:25.11"
+# container-digest:sha256:b04e8250f28479b2ea0f2c2b19187e1fad8bca0ad326b1b09dc569289152411d
 
-[scicode.tasks.chat.aa_scicode]
-required_env_vars = []
-
-
-###############################################################################
 [hle]
-container = "nvcr.io/nvidia/eval-factory/hle:25.10"
-
-[hle.tasks.chat.hle]
-required_env_vars = ["HF_TOKEN", "OPENAI_CLIENT_ID", "OPENAI_CLIENT_SECRET"]
+container = "nvcr.io/nvidia/eval-factory/hle:25.11"
+# container-digest:sha256:114e878ede1fd77d3e67bb046ae62a10e498cdc4a297f375d031880d220d63c4
 
-
-###############################################################################
 [bfcl]
-container = "nvcr.io/nvidia/eval-factory/bfcl:25.10"
-
-[bfcl.tasks.chat.bfclv2_ast_prompting]
-required_env_vars = []
-
-[bfcl.tasks.chat.bfclv3_ast_prompting]
-required_env_vars = []
+container = "nvcr.io/nvidia/eval-factory/bfcl:25.11"
+# container-digest:sha256:3926e91f1b2a0a1a639d44c6a93c05c6808ad121b79c1cd1131812e8984c44f2
 
-
-###############################################################################
 [profbench]
-container = "nvcr.io/nvidia/eval-factory/profbench:25.10"
-
-[profbench.tasks.chat.llm_judge]
-required_env_vars = []
-
-[profbench.tasks.chat.report_generation]
-required_env_vars = []
+container = "nvcr.io/nvidia/eval-factory/profbench:25.11"
+# container-digest:sha256:ce5455f06538b3408dbc7513e7c977a095521e35611020388252dee9527d83d5
 
-
-###############################################################################
 [vlmevalkit]
-container = "nvcr.io/nvidia/eval-factory/vlmevalkit:25.10"
-
-[vlmevalkit.tasks.vlm.ocrbench]
-required_env_vars = []
-
-[vlmevalkit.tasks.vlm.slidevqa]
-required_env_vars = ["OPENAI_CLIENT_ID", "OPENAI_CLIENT_SECRET"]
+container = "nvcr.io/nvidia/eval-factory/vlmevalkit:25.11"
+# container-digest:sha256:dee182f7052e17fe90ef1303c4c9ab6f0f67d2a9e159400555c74a9cc2bf8dfb
 
-[vlmevalkit.tasks.vlm.chartqa]
-required_env_vars = []
-
-[vlmevalkit.tasks.vlm.ai2d_judge]
-required_env_vars = ["OPENAI_CLIENT_ID", "OPENAI_CLIENT_SECRET"]
-
-
-###############################################################################
 [garak]
-container = "nvcr.io/nvidia/eval-factory/garak:25.10"
+container = "nvcr.io/nvidia/eval-factory/garak:25.11"
+# container-digest:sha256:a8d69f5582303921d0310a31ce8ed95d317a8d7750da32c35867f903315f0653
 
-[garak.tasks.chat.garak]
-required_env_vars = []
-
-###############################################################################
-# NOTE(wprazuch): to verify if the tasks need any env var setting
 [nemo_skills]
-container = "nvcr.io/nvidia/eval-factory/nemo_skills:25.10"
-
-[nemo_skills.tasks.chat.ns_aime2024]
-required_env_vars = ["JUDGE_API_KEY"]
-
-[nemo_skills.tasks.chat.ns_aime2025]
-required_env_vars = []
+container = "nvcr.io/nvidia/eval-factory/nemo_skills:25.11"
+# container-digest:sha256:36de6ac02d56ccda90ca8fd2bbf7c2dfd8a3318377273fdd5c9089d4a31fcec1
 
-[nemo_skills.tasks.chat.ns_bfcl_v3]
-required_env_vars = []
+[safety_eval]
+container = "nvcr.io/nvidia/eval-factory/safety-harness:25.11"
+# container-digest:sha256:08eeb3f5c3282522ca30da7d3ddc2cab1a48909be05ba561a0dae9a299c637f0
 
-[nemo_skills.tasks.chat.ns_gpqa]
-required_env_vars = ["HF_TOKEN"]
-
-[nemo_skills.tasks.chat.ns_hle]
-required_env_vars = []
-
-[nemo_skills.tasks.chat.ns_mmlu]
-required_env_vars = ["HF_TOKEN"]
-
-[nemo_skills.tasks.chat.ns_mmlu_pro]
-required_env_vars = ["HF_TOKEN"]
-
-[nemo_skills.tasks.chat.ns_aa_lcr]
-required_env_vars = ["JUDGE_API_KEY"]
-
-###############################################################################
-[safety-harness]
-container = "nvcr.io/nvidia/eval-factory/safety-harness:25.10"
-
-[safety-harness.tasks.chat.aegis_v2]
-required_env_vars = ["HF_TOKEN"]
-
-
-###############################################################################
-# NOTE(agronskiy): checked parity
 [helm]
-container = "nvcr.io/nvidia/eval-factory/helm:25.10"
-
-[helm.tasks.chat.medcalc_bench]
-
-[helm.tasks.chat.medec]
-
-[helm.tasks.chat.head_qa]
-
-[helm.tasks.chat.medbullets]
-
-[helm.tasks.chat.pubmed_qa]
+container = "nvcr.io/nvidia/eval-factory/helm:25.11"
+# container-digest:sha256:8147c9eaf4ef717b2de6390c264cb217e87206658fec4a1c730adb232b7a735c
 
-[helm.tasks.chat.ehr_sql]
-
-[helm.tasks.chat.race_based_med]
-
-[helm.tasks.chat.medhallu]
-
-[helm.tasks.chat.mtsamples_replicate]
-
-[helm.tasks.chat.aci_bench]
-
-[helm.tasks.chat.mtsamples_procedures]
-
-[helm.tasks.chat.medication_qa]
-
-[helm.tasks.chat.med_dialog_healthcaremagic]
-
-[helm.tasks.chat.med_dialog_icliniq]
-
-[helm.tasks.chat.medi_qa]
-
-
-###############################################################################
-# NOTE(agronskiy): checked parity
 [tooltalk]
-container = "nvcr.io/nvidia/eval-factory/tooltalk:25.10"
+container = "nvcr.io/nvidia/eval-factory/tooltalk:25.11"
+# container-digest:sha256:d829280eb563106a29a45020e20a9859db9f7e5f9e7988ec7a7d99277fb135ba
+
+[genai_perf_eval]
+container = "nvcr.io/nvidia/eval-factory/genai-perf:25.11"
+# container-digest:sha256:b6a3f60bc1762b2f2a216a5bdb73f84588e12c9914adcbdf0ecaa12d58acebe6
 
-[tooltalk.tasks.chat.tooltalk]
+[mmath]
+container = "nvcr.io/nvidia/eval-factory/mmath:25.11"
+# container-digest:sha256:4e30f8c40a7ff8f200734cf1251072b49888841bf0513fe2d5725ae81ab8b428

{nemo_evaluator_launcher-0.1.41.dist-info → nemo_evaluator_launcher-0.1.56.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.41
+Version: 0.1.56
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -467,6 +467,7 @@ License-File: LICENSE
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: jinja2<4.0.0,>=3.1.6
 Requires-Dist: leptonai>=0.25.0
+Requires-Dist: nemo-evaluator
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: requests>=2.32.4
 Requires-Dist: simple-parsing<0.2.0,>=0.1.7

{nemo_evaluator_launcher-0.1.41.dist-info → nemo_evaluator_launcher-0.1.56.dist-info}/RECORD

@@ -1,7 +1,7 @@
 nemo_evaluator_launcher/__init__.py,sha256=GT38zGwbvBOSeU52WCRx-n9N49LvLGEV1PItgKC8orA,2320
-nemo_evaluator_launcher/package_info.py,sha256=RGnruVj93NUhGw5fqI4DqujBesAV9Ixf05LquZO-JBU,1586
+nemo_evaluator_launcher/package_info.py,sha256=zHKOOOIbhAWOrU7pN-acMaSYwI2zdqC5syv_zy4x21A,1586
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
-nemo_evaluator_launcher/api/functional.py,sha256=4Se7avg6Wde4EK7is3y-KjXV5qR9RGBpe8zjwjej4M8,31859
+nemo_evaluator_launcher/api/functional.py,sha256=di5az6OkMSGBr5bWTT6JLgtqNBU8-fwAb_eZETyVulI,33641
 nemo_evaluator_launcher/api/types.py,sha256=W7ZQ9ZTPR6YxInxxsKE6NxuuQAg4pVYz6SRmFCFxY0A,3635
 nemo_evaluator_launcher/api/utils.py,sha256=q5HArRj7PKgBfeH3bOX8q1U97yMyQQp72yRRA5JP9PE,818
 nemo_evaluator_launcher/cli/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
@@ -10,17 +10,23 @@ nemo_evaluator_launcher/cli/info.py,sha256=2dZA2BqXpTG1wO_Wzt6Ol9ZNJzJJ0PibOB0hL
 nemo_evaluator_launcher/cli/kill.py,sha256=C-4PWmMu8mIITo92o5AHxtq_s-8Cckbp7wAlG0I_ylw,1323
 nemo_evaluator_launcher/cli/logs.py,sha256=eYgfDbHNZhw1xo56bwCIR1sGQYJIPPzzUJu-pqseDWk,3774
 nemo_evaluator_launcher/cli/ls_runs.py,sha256=vJTwRdhVKLolnJuP8AnnQdJBE-BKZfCcCypLKSz5gqs,4942
-nemo_evaluator_launcher/cli/ls_tasks.py,sha256=Pd2lBQOQBNHBWrjk4tZg0SQ9Ul9F2Ak-zOyh-G9x-DY,5293
-nemo_evaluator_launcher/cli/main.py,sha256=iNEAeE7_dfDia4oCjUYhYikJJJXg9cMlbGADwMfMtfM,7613
-nemo_evaluator_launcher/cli/run.py,sha256=XKF4nRdijhFH01JvMtWXav6tc6Xp7tAhbSnk32q8Ma4,9966
+nemo_evaluator_launcher/cli/ls_task.py,sha256=sChh9UQJK0Zh1g3RitVtm8o_6qJQuS75JGrHXf3Y0y4,10954
+nemo_evaluator_launcher/cli/ls_tasks.py,sha256=aMzuAiUUeTCeSN3XCtoneOm-BBSnOVfhgIzgxZ8Y8wI,11167
+nemo_evaluator_launcher/cli/main.py,sha256=3BwJXuB9WHmiTKu4FR2dyrVonwpo8YWcPfw3MSmqOu8,8148
+nemo_evaluator_launcher/cli/run.py,sha256=KGBIXiDfT5OUv09FaTl-GyOxDhjxWEIE1rGlUkccB2o,11336
 nemo_evaluator_launcher/cli/status.py,sha256=ANdu0JYnfKNvd1gXmdu_0FrbPG-g0A_R4leOuNXzenQ,5947
 nemo_evaluator_launcher/cli/version.py,sha256=GgMNTAd4S0bu3t-uVfVedAf7p6pymWDDwOaNm4WHOxQ,1998
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=WPzg5Iu2ojvFpBuYahSt3voP_iEUpoO8NgqMLUBwFxA,9767
-nemo_evaluator_launcher/common/helpers.py,sha256=MrLHPlOSn8GGc7sxB0xlDmUN0uWluSgVmDLUBwuo7pU,13540
+nemo_evaluator_launcher/common/helpers.py,sha256=pWYd1P5tbBSAK1JPv7dk_d_Sq92gJ4NIHXiiOEyNiZY,13218
 nemo_evaluator_launcher/common/logging_utils.py,sha256=7QkWlpA80QN5ipTUFJ198IiAsPRS36C6ISAAtNverbA,12338
-nemo_evaluator_launcher/common/mapping.py,sha256=tD3jWN7rm9-iJEFlENhYMt7adz8DKs67g3Xd43XIAMM,10731
-nemo_evaluator_launcher/common/printing_utils.py,sha256=QPjLqFlp0dd18RY__UYo5-yG8OqABaa78cpzMhZVCeo,2579
+nemo_evaluator_launcher/common/mapping.py,sha256=O4Xg_0hgnQOOxAEGFlFJjHh_sTXvbIEUCgNXYxvfnDU,18035
+nemo_evaluator_launcher/common/printing_utils.py,sha256=K9_-ENUIGHcL9i-fmGscfCJoV-uWJlaMKNToNx1Ypmg,2631
+nemo_evaluator_launcher/common/container_metadata/__init__.py,sha256=QxPpIUTWprJybsfsaNoHnpLNIGxIVERW0mAtZ-u-F3c,2145
+nemo_evaluator_launcher/common/container_metadata/intermediate_repr.py,sha256=aa9J0tdvdVYpGPu85vkdpBnFOpSReLdA0SNb45jAC0Y,17525
+nemo_evaluator_launcher/common/container_metadata/loading.py,sha256=KY6mML3St4q-OgDQ1TeSyWQRIGSUx-CvLSC1sHjI1jc,41561
+nemo_evaluator_launcher/common/container_metadata/registries.py,sha256=UNJdUBiXJyMeKKSIgCHZod4bDAHGhOFdXH9WjlQJbKw,28404
+nemo_evaluator_launcher/common/container_metadata/utils.py,sha256=kaINUugS7Jhydm7wcYrOQujMe7SfY8dkaRzzr89Z5aE,2331
 nemo_evaluator_launcher/configs/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/configs/default.yaml,sha256=JHFjSl3KByhggRMTTo9nesQATVoz18PJIV6KM5Wng64,974
 nemo_evaluator_launcher/configs/deployment/generic.yaml,sha256=8_Z0fcjZuH6GfV9jJkY_8CS18Tbsn0gV-nK1LfGr_Vg,1262
@@ -37,13 +43,13 @@ nemo_evaluator_launcher/executors/base.py,sha256=QC1HDyTqXJFQMPMz48A-DLOFu3le4NF
 nemo_evaluator_launcher/executors/registry.py,sha256=8QXSrsJyHeNi8iSttJ8KWQLXmZve1vxnnCNw_CkeopI,1409
 nemo_evaluator_launcher/executors/lepton/__init__.py,sha256=F_7yuBaYQ6WWTcptADdkL3AIZ_jXJQHGgKag-Hm7BbQ,698
 nemo_evaluator_launcher/executors/lepton/deployment_helpers.py,sha256=vzAWaNmacRsloURwVIc6LsP-xpKwvr7vsPlFIFTFdLQ,23234
-nemo_evaluator_launcher/executors/lepton/executor.py,sha256=gKls0DNTVyFVOh3rjIPAVruaAeQ5qmVcNoZctlLnb5w,44347
+nemo_evaluator_launcher/executors/lepton/executor.py,sha256=CG4bRnF3BNXujgztsX15WmMFsnmcfalqrW8K_z95phI,45049
 nemo_evaluator_launcher/executors/lepton/job_helpers.py,sha256=6baTxcygfP1oFgAJ7I9EL4xRlcJDWqbqzZoE1CRrwSk,13528
 nemo_evaluator_launcher/executors/local/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/local/executor.py,sha256=ElRNKv8GVlcOHP23Uc67RiSleI6GfWpBK0e8w6VcXRk,36880
+nemo_evaluator_launcher/executors/local/executor.py,sha256=lLBEv0UoB7PRKdVsYdcodW0klCn90t0ua3Sp41IZr4U,37012
 nemo_evaluator_launcher/executors/local/run.template.sh,sha256=S6qLqnqA3B4P_-ngklCU5dO9rKV0Qom5NWDwDHf5i0g,6103
 nemo_evaluator_launcher/executors/slurm/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/slurm/executor.py,sha256=2miKvU5kHr5fs706MxgAjDPE8nuqmOuWjnzni2F97qo,55108
+nemo_evaluator_launcher/executors/slurm/executor.py,sha256=zumFTbnKQzKNv5g7Y_PxsAXAsHIIvXKvIIOPLS8hoiA,59726
 nemo_evaluator_launcher/executors/slurm/proxy.cfg.template,sha256=nPg_JyBHPcjI1RSxDvl-lBgcbakrbZLl4IAXNBc8LM0,594
 nemo_evaluator_launcher/exporters/__init__.py,sha256=mBXG9FG48FeYrs8sF0zA2mgo1eqBmRgoml7zjJrqDso,1323
 nemo_evaluator_launcher/exporters/base.py,sha256=0BEqS-Zjez-KsrGE9yfo8S5w2uwMW3btBZve3SiiUp0,4307
@@ -53,10 +59,11 @@ nemo_evaluator_launcher/exporters/mlflow.py,sha256=qFEe9774s31Uybpah7kiz4BfZUDMy
 nemo_evaluator_launcher/exporters/registry.py,sha256=XsPTv_SBAFjcErO6BJ3OHqs3EvXQpLeyKRJuK9Ql4_M,1299
 nemo_evaluator_launcher/exporters/utils.py,sha256=5HGBKoFTSCng-GhAJpvfAa7N52_r9UDyrGlBLqRiiGo,22587
 nemo_evaluator_launcher/exporters/wandb.py,sha256=1qRUV_YE1Ury7rH7KH65AabR7gmEQ38kXBh2XrfiEpE,18082
-nemo_evaluator_launcher/resources/mapping.toml,sha256=oUeYpfOTyh5iiQs1rdc9zo-j_1P8ReWAzole2wYDFgg,12370
-nemo_evaluator_launcher-0.1.41.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.41.dist-info/METADATA,sha256=rCHDM0bplzE1q9gmi1GYzG1oQBpia3NfToSOEmN-JiU,28730
-nemo_evaluator_launcher-0.1.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.41.dist-info/entry_points.txt,sha256=HPmLybw-y1y4NxjK7slFS4mtTB7p4pnjRvCtJLvSiZs,174
-nemo_evaluator_launcher-0.1.41.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.41.dist-info/RECORD,,
+nemo_evaluator_launcher/resources/all_tasks_irs.yaml,sha256=g2T5Jb_2zkJFZd4gqHmOLKiKR52ioz9adjOYcaSEnyE,978316
+nemo_evaluator_launcher/resources/mapping.toml,sha256=iB5tHeMYJMnnej4TwKmrmI1rq3zJ-rWymO8eWxvfGvk,3619
+nemo_evaluator_launcher-0.1.56.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.56.dist-info/METADATA,sha256=uKG_JgnSQcHFxj2Nxs-Qmq7jc39BCksOdQ-qMfevGUE,28760
+nemo_evaluator_launcher-0.1.56.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.56.dist-info/entry_points.txt,sha256=HPmLybw-y1y4NxjK7slFS4mtTB7p4pnjRvCtJLvSiZs,174
+nemo_evaluator_launcher-0.1.56.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.56.dist-info/RECORD,,