nemo-evaluator-launcher 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl

nemo_evaluator_launcher/common/printing_utils.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Printing utils for more structured or visually appealing prints.
+
+NOTE: use printing only for main application output that matters. For logging,
+see `logging_utils.py`.
+
+USAGE:
+```
+  from nemo_evaluator_launcher.common.printing_utils import red, bold
+  print(bold(red("some red bold")))
+```
+
+
+"""
+
+import os
+
+# If this env var is set, it will override a more standard "LOG_LEVEL". If
+# both are unset, default would be used.
+_DISABLE_COLOR_ENV_VAR = "NEMO_EVALUATOR_DISABLE_COLOR"
+
+
+def _is_color_disabled():
+    env_var = os.environ.get(_DISABLE_COLOR_ENV_VAR, "0").lower()
+
+    if "1" in env_var or "yes" in env_var or "y" in env_var or "true" in env_var:
+        return True
+
+    return False
+
+
+_CODES: dict[str, str] = dict(
+    green="\033[32m",
+    red="\033[31m",
+    red_bg="\033[41m",  # red background
+    cyan="\033[36m",
+    yellow="\033[33m",
+    magenta="\033[35m",
+    grey="\033[90m",
+    bold="\033[1m",
+    reset="\033[0m",
+)
+
+# If the colors are disabled, we null-out all the codes.
+if _is_color_disabled():
+    for c in _CODES.keys():
+        _CODES[c] = ""
+
+
+def green(s: str) -> str:
+    return _CODES["green"] + s + _CODES["reset"]
+
+
+def red(s: str) -> str:
+    return _CODES["red"] + s + _CODES["reset"]
+
+
+def red_bg(s: str) -> str:
+    return _CODES["red_bg"] + s + _CODES["reset"]
+
+
+def cyan(s: str) -> str:
+    return _CODES["cyan"] + s + _CODES["reset"]
+
+
+def yellow(s: str) -> str:
+    return _CODES["yellow"] + s + _CODES["reset"]
+
+
+def magenta(s: str) -> str:
+    return _CODES["magenta"] + s + _CODES["reset"]
+
+
+def grey(s: str) -> str:
+    return _CODES["grey"] + s + _CODES["reset"]
+
+
+def bold(s: str) -> str:
+    return _CODES["bold"] + s + _CODES["reset"]

nemo_evaluator_launcher/configs/execution/slurm/default.yaml

@@ -14,16 +14,17 @@
 # limitations under the License.
 #
 # Each slurm cluster has its own flavour, below we provide some defaults that might meet one's needs.
-hostname: ???
-username: ${oc.env:USER}
-account: ???
+type: slurm              # Executor is chosen based on this field
+hostname: ???            # SLURM headnode (login) hostname (required)
+username: ${oc.env:USER} # Defaults to $USER env var
+account: ???             # SLURM account allocation (required)
+output_dir: ???          # Absolute path accessible on compute nodes (required)
 partition: batch
 num_nodes: 1
 ntasks_per_node: 1
 gres: gpu:8
 walltime: 01:00:00
 subproject: nemo-evaluator-launcher
-output_dir: ???
 env_vars:
   deployment: {}
   evaluation: {}

nemo_evaluator_launcher/executors/lepton/executor.py

@@ -78,9 +78,32 @@ class LeptonExecutor(BaseExecutor):
                 "LeptonExecutor supports deployment types: 'vllm', 'sglang', 'nim', 'none'"
             )
 
+        # Load tasks mapping
+        tasks_mapping = load_tasks_mapping()
+        job_ids = []
+        lepton_job_names = []
+        endpoint_names = []  # Track multiple endpoints
+        db = ExecutionDB()
+
         # Generate invocation ID
         invocation_id = generate_invocation_id()
 
+        # DRY-RUN mode
+        if dry_run:
+            output_dir = Path(cfg.execution.output_dir).absolute() / invocation_id
+            output_dir.mkdir(parents=True, exist_ok=True)
+
+            # Validate configuration
+            _dry_run_lepton(cfg, tasks_mapping, invocation_id=invocation_id)
+
+            if cfg.deployment.type == "none":
+                print("Using existing endpoint (deployment: none)")
+                print("using shared endpoint")
+            else:
+                print(f"with endpoint type '{cfg.deployment.type}'")
+
+            return invocation_id
+
         # For deployment: none, we use the existing endpoint for all tasks
         if cfg.deployment.type == "none":
             print("📌 Using existing endpoint (deployment: none)")
@@ -88,13 +111,6 @@ class LeptonExecutor(BaseExecutor):
             print(f"✅ Using shared endpoint: {shared_endpoint_url}")
 
         try:
-            # Load tasks mapping
-            tasks_mapping = load_tasks_mapping()
-            job_ids = []
-            lepton_job_names = []
-            endpoint_names = []  # Track multiple endpoints
-            db = ExecutionDB()
-
             # Create local directory for outputs
             output_dir = Path(cfg.execution.output_dir).absolute() / invocation_id
             output_dir.mkdir(parents=True, exist_ok=True)
@@ -139,8 +155,13 @@ class LeptonExecutor(BaseExecutor):
                     task_index = str(idx)
                     endpoint_name = f"{cfg.deployment.type}-{short_task_name}-{task_index}-{short_invocation}"
 
-                    # Ensure we don't exceed 36 character limit
                     if len(endpoint_name) > 36:
+                        logger.info(
+                            "Lepton endpoint name will be deployed under name {task_name}",
+                            task_name=task.name,
+                            original=endpoint_name,
+                            limit=36,
+                        )
                         # Truncate task name further if needed
                         max_task_len = (
                             36
@@ -151,7 +172,19 @@ class LeptonExecutor(BaseExecutor):
                         )  # 3 hyphens
                         short_task_name = sanitized_task_name[:max_task_len]
                         endpoint_name = f"{cfg.deployment.type}-{short_task_name}-{task_index}-{short_invocation}"
+                        logger.info(
+                            "Lepton endpoint name is auto-generated",
+                            task_name=task.name,
+                            original=endpoint_name,
+                            truncated=endpoint_name,
+                            limit=36,
+                        )
 
+                    logger.info(
+                        "Lepton endpoint name (auto-generated)",
+                        task_name=task.name,
+                        endpoint_name=endpoint_name,
+                    )
                     endpoint_names.append(endpoint_name)
                     endpoint_creation_tasks.append((idx, task, endpoint_name))
 
@@ -298,20 +331,6 @@ class LeptonExecutor(BaseExecutor):
                     f"✅ All {len(cfg.evaluation.tasks)} endpoints created successfully!"
                 )
 
-            if dry_run:
-                print("🔍 DRY RUN: Lepton job configurations prepared")
-                print(f"   - Tasks: {len(cfg.evaluation.tasks)}")
-                for idx, task in enumerate(cfg.evaluation.tasks):
-                    if cfg.deployment.type == "none":
-                        print(f"   - Task {idx}: {task.name} using shared endpoint")
-                    else:
-                        print(
-                            f"   - Task {idx}: {task.name} with endpoint {endpoint_names[idx]}"
-                        )
-                print(f"   - Output directory: {output_dir}")
-                print("\nTo submit jobs, run the executor without --dry-run")
-                return invocation_id
-
             # ================================================================
             # JOB SUBMISSION (Sequential, as before)
             # ================================================================
@@ -334,8 +353,18 @@ class LeptonExecutor(BaseExecutor):
                 max_base_length = 36 - 1 - len(suffix)  # -1 for the hyphen
                 if len(base_job_name) > max_base_length:
                     base_job_name = base_job_name[:max_base_length]
+                    logger.info(
+                        "Lepton job auto-generated name",
+                        task_name=task.name,
+                        job_name=f"{base_job_name}-{suffix}",
+                    )
 
                 lepton_job_name = f"{base_job_name}-{suffix}"
+                logger.info(
+                    "Lepton job name (auto-generated)",
+                    task_name=task.name,
+                    job_name=lepton_job_name,
+                )
                 job_ids.append(job_id)
                 lepton_job_names.append(lepton_job_name)
 
@@ -377,7 +406,12 @@ class LeptonExecutor(BaseExecutor):
                     cfg.target.api_endpoint.url = full_endpoint_url
 
                     # Generate command with the correct endpoint URL
-                    eval_command = get_eval_factory_command(cfg, task, task_definition)
+                    eval_command_struct = get_eval_factory_command(
+                        cfg, task, task_definition
+                    )
+                    eval_command = eval_command_struct.cmd
+                    # Debug string for explainability of some base64-parts of the command
+                    eval_command_debug_comment = eval_command_struct.debug
 
                 finally:
                     # Restore original URL and struct mode
@@ -402,6 +436,7 @@ class LeptonExecutor(BaseExecutor):
                     task_name=task.name,
                     invocation_id=invocation_id,
                     eval_command=eval_command,  # Pass the fixed command
+                    eval_command_debug_comment=eval_command_debug_comment,
                 )
 
                 # Prepare job command to run the launch script
@@ -705,6 +740,7 @@ def _create_evaluation_launch_script(
     task_name: str,
     invocation_id: str,
     eval_command: str,
+    eval_command_debug_comment: str,
 ) -> str:
     """Create bash script for running evaluation in Lepton job container.
 
@@ -718,6 +754,7 @@ def _create_evaluation_launch_script(
         task_name: Name of the evaluation task.
         invocation_id: Unique invocation identifier.
         eval_command: The evaluation command with correct endpoint URL.
+        eval_command_debug_comment: The debug comment for placing into the script and easy debug
 
     Returns:
         String containing the bash launch script.
@@ -750,6 +787,8 @@ echo "Invocation ID: {invocation_id}"
 echo "Endpoint URL: {endpoint_url}"
 echo "Command: {eval_command_modified}"
 
+{eval_command_debug_comment}
+
 # Execute the evaluation with proper error handling
 set +e
 {eval_command_modified}
@@ -773,6 +812,82 @@ exit 0
     return script
 
 
+def _dry_run_lepton(
+    cfg: DictConfig, tasks_mapping: dict, invocation_id: str | None = None
+) -> None:
+    print("DRY RUN: Lepton job configurations prepared")
+    try:
+        # validate tasks
+        for task in cfg.evaluation.tasks:
+            get_task_from_mapping(task.name, tasks_mapping)
+
+        # nice-to-have checks (existing endpoint URL or endpoints mapping)
+        if getattr(cfg.deployment, "type", None) == "none":
+            tgt = getattr(cfg, "target", {})
+            api = (
+                tgt.get("api_endpoint")
+                if isinstance(tgt, dict)
+                else getattr(tgt, "api_endpoint", None)
+            ) or {}
+            url = api.get("url") if isinstance(api, dict) else getattr(api, "url", None)
+            if not url or not str(url).strip():
+                raise ValueError(
+                    "target.api_endpoint.url must be set when deployment.type == 'none'"
+                )
+        else:
+            endpoints_cfg = getattr(cfg.deployment, "endpoints", {}) or {}
+            for task in cfg.evaluation.tasks:
+                td = get_task_from_mapping(task.name, tasks_mapping)
+                etype = td.get("endpoint_type")
+                if etype not in endpoints_cfg:
+                    raise ValueError(
+                        f"deployment.endpoints missing path for endpoint_type '{etype}' (task '{task.name}')"
+                    )
+                path = endpoints_cfg.get(etype)
+                if not isinstance(path, str) or not path.startswith("/"):
+                    raise ValueError(
+                        f"deployment.endpoints['{etype}'] must be a non-empty path starting with '/'"
+                    )
+
+        # lepton env var presence (reference-level)
+        tasks_cfg = getattr(cfg.execution, "lepton_platform", {}).get("tasks", {}) or {}
+        lepton_env_vars = tasks_cfg.get("env_vars", {}) or {}
+        api_key_name = getattr(
+            getattr(cfg, "target", {}).get("api_endpoint", {}), "api_key_name", None
+        )
+        for task in cfg.evaluation.tasks:
+            td = get_task_from_mapping(task.name, tasks_mapping)
+            required = td.get("required_env_vars", []) or []
+            for var in required:
+                if var == "API_KEY":
+                    if not (("API_KEY" in lepton_env_vars) or bool(api_key_name)):
+                        raise ValueError(
+                            f"Task '{task.name}' requires API_KEY: set execution.lepton_platform.tasks.env_vars.API_KEY "
+                            "or target.api_endpoint.api_key_name"
+                        )
+                else:
+                    if var not in lepton_env_vars:
+                        raise ValueError(
+                            f"Task '{task.name}' requires {var}: set it under execution.lepton_platform.tasks.env_vars"
+                        )
+
+        # success (use realized output directory if invocation_id is available)
+        preview_output_dir = (
+            Path(cfg.execution.output_dir).absolute() / invocation_id
+            if invocation_id
+            else Path(cfg.execution.output_dir).absolute() / "<invocation_id>"
+        )
+        print(f"   - Tasks: {len(cfg.evaluation.tasks)}")
+        for idx, task in enumerate(cfg.evaluation.tasks):
+            print(f"   - Task {idx}: {task.name}")
+        print(f"   - Output directory: {preview_output_dir}")
+        print("\nTo run evaluation, execute run command without --dry-run")
+    except Exception as e:
+        print(f"❌ Configuration invalid: {e}")
+        logger.error("Lepton dry-run validation failed", error=str(e))
+        return
+
+
 def _get_statuses_for_invocation_id(id: str, db: ExecutionDB) -> List[ExecutionStatus]:
     """Helper method that returns statuses if id is the invocation id"""
     jobs = db.get_jobs(id)

nemo_evaluator_launcher/executors/local/executor.py

@@ -47,6 +47,7 @@ from nemo_evaluator_launcher.common.mapping import (
     get_task_from_mapping,
     load_tasks_mapping,
 )
+from nemo_evaluator_launcher.common.printing_utils import bold, cyan, grey
 from nemo_evaluator_launcher.executors.base import (
     BaseExecutor,
     ExecutionState,
@@ -155,6 +156,16 @@ class LocalExecutor(BaseExecutor):
 
             task_output_dir = output_dir / task.name
             task_output_dir.mkdir(parents=True, exist_ok=True)
+            eval_factory_command_struct = get_eval_factory_command(
+                cfg, task, task_definition
+            )
+            eval_factory_command = eval_factory_command_struct.cmd
+            # The debug comment for placing into the script and easy debug. Reason
+            # (see `CmdAndReadableComment`) is the current way of passing the command
+            # is base64-encoded config `echo`-ed into file.
+            # TODO(agronskiy): cleaner way is to encode everything with base64, not
+            # some parts (like ef_config.yaml) and just output as logs somewhere.
+            eval_factory_command_debug_comment = eval_factory_command_struct.debug
             evaluation_task = {
                 "name": task.name,
                 "job_id": job_id,
@@ -162,9 +173,8 @@ class LocalExecutor(BaseExecutor):
                 "container_name": container_name,
                 "env_vars": env_vars,
                 "output_dir": task_output_dir,
-                "eval_factory_command": get_eval_factory_command(
-                    cfg, task, task_definition
-                ),
+                "eval_factory_command": eval_factory_command,
+                "eval_factory_command_debug_comment": eval_factory_command_debug_comment,
             }
             evaluation_tasks.append(evaluation_task)
 
@@ -198,23 +208,28 @@ class LocalExecutor(BaseExecutor):
         )
 
         if dry_run:
-            print("\n\n=============================================\n\n")
-            print(f"DRY RUN: Scripts prepared and saved to {output_dir}")
+            print(bold("\n\n=============================================\n\n"))
+            print(bold(cyan(f"DRY RUN: Scripts prepared and saved to {output_dir}")))
             if is_execution_mode_sequential:
                 print(
-                    "\n\n =========== Main script | run_all.sequential.sh ===================== \n\n"
+                    cyan(
+                        "\n\n=========== Main script | run_all.sequential.sh =====================\n\n"
+                    )
                 )
+
                 with open(output_dir / "run_all.sequential.sh", "r") as f:
-                    print(f.read())
+                    print(grey(f.read()))
             else:
                 for idx, task in enumerate(cfg.evaluation.tasks):
                     task_output_dir = output_dir / task.name
                     print(
-                        f"\n\n =========== Task script | {task.name}/run.sh ===================== \n\n"
+                        cyan(
+                            f"\n\n=========== Task script | {task.name}/run.sh =====================\n\n"
+                        )
                     )
                     with open(task_output_dir / "run.sh", "r") as f:
-                        print(f.read())
-            print("\nTo execute, run without --dry-run")
+                        print(grey(f.read()))
+            print(bold("\nTo execute, run without --dry-run"))
             return invocation_id
 
         # Save launched jobs metadata
@@ -284,13 +299,13 @@ class LocalExecutor(BaseExecutor):
                 error_msg = f"Script for {name} exited with code {exit_code}"
                 raise RuntimeError(f"Job startup failed | {error_msg}")
 
-        print("\nCommands for real-time monitoring:")
+        print(bold(cyan("\nCommands for real-time monitoring:")))
         for job_id, evaluation_task in zip(job_ids, evaluation_tasks):
             log_file = evaluation_task["output_dir"] / "logs" / "stdout.log"
             print(f"  tail -f {log_file}")
 
-        print("\nFollow all logs for this invocation:")
-        print(f"  tail -f {output_dir}/*/logs/stdout.log")
+        print(bold(cyan("\nFollow all logs for this invocation:")))
+        print(f"  tail -f {output_dir}/*/logs/stdout.log\n")
 
         return invocation_id
 

nemo_evaluator_launcher/executors/local/run.template.sh

@@ -40,6 +40,9 @@ else
     # Create pre-start stage file
     echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.pre-start"
 
+    # Debug contents of the eval factory command's config
+    {{ task.eval_factory_command_debug_comment | indent(4) }}
+
     # Docker run with eval factory command
     (
         echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.running"
@@ -51,7 +54,7 @@ else
       {% endfor -%}
       {{ task.eval_image }} \
       bash -c '
-        {{ task.eval_factory_command }} ;
+        {{ task.eval_factory_command | indent(8) }} ;
         exit_code=$?
         chmod 777 -R /results;
         if [ "$exit_code" -ne 0 ]; then

nemo_evaluator_launcher/executors/slurm/executor.py

@@ -50,6 +50,7 @@ from nemo_evaluator_launcher.common.mapping import (
     get_task_from_mapping,
     load_tasks_mapping,
 )
+from nemo_evaluator_launcher.common.printing_utils import bold, cyan, grey
 from nemo_evaluator_launcher.executors.base import (
     BaseExecutor,
     ExecutionState,
@@ -130,13 +131,13 @@ class SlurmExecutor(BaseExecutor):
                 remote_runsub_paths.append(remote_runsub_path)
 
             if dry_run:
-                print("\n\n=============================================\n\n")
-                print("DRY RUN: SLURM scripts prepared")
+                print(bold("\n\n=============================================\n\n"))
+                print(bold(cyan("DRY RUN: SLURM scripts prepared")))
                 for idx, local_runsub_path in enumerate(local_runsub_paths):
-                    print(f"\n\n =========== Task {idx} ===================== \n\n")
+                    print(cyan(f"\n\n=========== Task {idx} =====================\n\n"))
                     with open(local_runsub_path, "r") as f:
-                        print(f.read())
-                print("\nTo submit jobs, run the executor without --dry-run")
+                        print(grey(f.read()))
+                print(bold("To submit jobs") + ", run the executor without --dry-run")
                 return invocation_id
 
             socket = str(Path(tmpdirname) / "socket")
@@ -589,7 +590,20 @@ def _create_slurm_sbatch_script(
     ):
         evaluation_mounts_list.append(f"{source_mnt}:{target_mnt}")
 
+    eval_factory_command_struct = get_eval_factory_command(cfg, task, task_definition)
+    eval_factory_command = eval_factory_command_struct.cmd
+    # The debug comment for placing into the script and easy debug. Reason
+    # (see `CmdAndReadableComment`) is the current way of passing the command
+    # is base64-encoded config `echo`-ed into file.
+    # TODO(agronskiy): cleaner way is to encode everything with base64, not
+    # some parts (like ef_config.yaml) and just output as logs somewhere.
+    eval_factory_command_debug_comment = eval_factory_command_struct.debug
+
     # add evaluation srun command
+    s += "# Debug contents of the eval factory command's config\n"
+    s += eval_factory_command_debug_comment
+    s += "\n\n"
+
     s += "# evaluation client\n"
     s += "srun --mpi pmix --overlap "
     s += "--container-image {} ".format(eval_image)
@@ -600,10 +614,11 @@ def _create_slurm_sbatch_script(
         s += "--container-env {} ".format(",".join(evaluation_env_var_names))
     if not cfg.execution.get("mounts", {}).get("mount_home", True):
         s += "--no-container-mount-home "
+
     s += "--container-mounts {} ".format(",".join(evaluation_mounts_list))
     s += "--output {} ".format(remote_task_subdir / "logs" / "client-%A.out")
-    s += "bash -c '"
-    s += get_eval_factory_command(cfg, task, task_definition)
+    s += "bash -c '\n"
+    s += eval_factory_command
     s += "'\n\n"
 
     # terminate the server after all evaluation clients finish

nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 16
+PATCH = 18
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator_launcher-0.1.16.dist-info → nemo_evaluator_launcher-0.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.16
+Version: 0.1.18
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.16.dist-info → nemo_evaluator_launcher-0.1.18.dist-info}/RECORD

@@ -1,24 +1,25 @@
 nemo_evaluator_launcher/__init__.py,sha256=2F703fttLaIyMHoVD54rptHMXt4AWnplHDrwWJ3e3PM,1930
-nemo_evaluator_launcher/package_info.py,sha256=T4Fbaynh6kLoILH0aWiwq9Y2R_lCn62kYI_Bb8LA2p0,1586
+nemo_evaluator_launcher/package_info.py,sha256=praNLExAuGkNXRb2vVR5Kn_A9aqyBy60Pwx9DiOg-Pk,1586
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
 nemo_evaluator_launcher/api/functional.py,sha256=T1HTIeiTXb-APWP7lPPTwFam4vFOApZCScRi6tMp538,27648
-nemo_evaluator_launcher/api/types.py,sha256=RXr_QoKdhejj1T9-HybSjd4KTxJmSv0bE0uLUFtF7Zc,3269
+nemo_evaluator_launcher/api/types.py,sha256=W7ZQ9ZTPR6YxInxxsKE6NxuuQAg4pVYz6SRmFCFxY0A,3635
 nemo_evaluator_launcher/api/utils.py,sha256=q5HArRj7PKgBfeH3bOX8q1U97yMyQQp72yRRA5JP9PE,818
 nemo_evaluator_launcher/cli/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/cli/debug.py,sha256=uJurq0yWzUnMy3S2hCX6msksPtQAQnon11GJlWfy_-4,16506
 nemo_evaluator_launcher/cli/export.py,sha256=GRXxusKDq_1qjMKN6MKOIjZ8x4u5ERgXwHSAGrvsGCY,11211
+nemo_evaluator_launcher/cli/info.py,sha256=2dZA2BqXpTG1wO_Wzt6Ol9ZNJzJJ0PibOB0hLFZpL14,20705
 nemo_evaluator_launcher/cli/kill.py,sha256=C-4PWmMu8mIITo92o5AHxtq_s-8Cckbp7wAlG0I_ylw,1323
 nemo_evaluator_launcher/cli/ls_runs.py,sha256=vJTwRdhVKLolnJuP8AnnQdJBE-BKZfCcCypLKSz5gqs,4942
 nemo_evaluator_launcher/cli/ls_tasks.py,sha256=Pd2lBQOQBNHBWrjk4tZg0SQ9Ul9F2Ak-zOyh-G9x-DY,5293
-nemo_evaluator_launcher/cli/main.py,sha256=23PJXum3bjfZryiH6HJfhX0AbDsqpq2dnemZkt6Qc9U,7245
-nemo_evaluator_launcher/cli/run.py,sha256=vxSL0Sb-h9v2EErlUtwcn_8HVSA2N96EcwFHwcoGTSc,6785
-nemo_evaluator_launcher/cli/status.py,sha256=3kv-VHMHlD2qHwgKzgmTZ5x_D1c9-rW86hBOghHk35M,6108
+nemo_evaluator_launcher/cli/main.py,sha256=MQo-DcVF_f0aBgnVUic5PP1qlfXXs0Evkd1X4EXdOvA,7217
+nemo_evaluator_launcher/cli/run.py,sha256=3-C_GvaIg9IxqpvC4P3h3lHcHdjB94Zpgq0ccOXVcpw,7503
+nemo_evaluator_launcher/cli/status.py,sha256=ANdu0JYnfKNvd1gXmdu_0FrbPG-g0A_R4leOuNXzenQ,5947
 nemo_evaluator_launcher/cli/version.py,sha256=puMwIvkmfD3HESjftdTSP6T3Nc8J4cbz8uXWHJcTemY,2030
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=WPzg5Iu2ojvFpBuYahSt3voP_iEUpoO8NgqMLUBwFxA,9767
-nemo_evaluator_launcher/common/helpers.py,sha256=YG1mQNijKZkw7mqTPhU3yatgWzufXyyKc6oSxuhJpLs,7710
+nemo_evaluator_launcher/common/helpers.py,sha256=ZB4gTJQLGD33Rw93phFxsFnS466SUyMZdcsiZ1pBlUk,8842
 nemo_evaluator_launcher/common/logging_utils.py,sha256=8UMAQ22t5NAJRDZtI0gVbdKUlNAiG23WQwZZ0HwzOT4,11843
 nemo_evaluator_launcher/common/mapping.py,sha256=tD3jWN7rm9-iJEFlENhYMt7adz8DKs67g3Xd43XIAMM,10731
+nemo_evaluator_launcher/common/printing_utils.py,sha256=YICPY-KhxjL5QNEFJNvYfnj6_ArkZURDdP1pizqY-yU,2368
 nemo_evaluator_launcher/configs/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/configs/default.yaml,sha256=JHFjSl3KByhggRMTTo9nesQATVoz18PJIV6KM5Wng64,974
 nemo_evaluator_launcher/configs/deployment/generic.yaml,sha256=8_Z0fcjZuH6GfV9jJkY_8CS18Tbsn0gV-nK1LfGr_Vg,1262
@@ -29,19 +30,19 @@ nemo_evaluator_launcher/configs/deployment/trtllm.yaml,sha256=nZF1ueCF9uMticElge
 nemo_evaluator_launcher/configs/deployment/vllm.yaml,sha256=NOAfK7hALTeVxjNJ63zEkb0vnefeRuAQCh6OZwe18pA,1476
 nemo_evaluator_launcher/configs/execution/local.yaml,sha256=0XtVXHeCK-zAoTURxnf4pr9RCe46HUCpKVzcfPlskz4,740
 nemo_evaluator_launcher/configs/execution/lepton/default.yaml,sha256=SRSMxohEtafzb-QS_oz4kP-RBgigRCAZgYwKkXtjymY,2930
-nemo_evaluator_launcher/configs/execution/slurm/default.yaml,sha256=szjDrKMC4ZAN8CO-KMsnFkc7-Vi6o6U8SSLYYZJHPYs,1064
+nemo_evaluator_launcher/configs/execution/slurm/default.yaml,sha256=zTevKSKch7yl4D0wGh4BjDBuI0YAiJSljVKwD7G8Cww,1328
 nemo_evaluator_launcher/executors/__init__.py,sha256=mSU1op5r7R_vqOCLDP84z6utfFgXOIl_1vBzN7KOC6o,1042
 nemo_evaluator_launcher/executors/base.py,sha256=4BOz2-jMG1OJ-5o5qCh-SJqLUE64YJWlnmB9hc1p4Pc,4040
 nemo_evaluator_launcher/executors/registry.py,sha256=8QXSrsJyHeNi8iSttJ8KWQLXmZve1vxnnCNw_CkeopI,1409
 nemo_evaluator_launcher/executors/lepton/__init__.py,sha256=F_7yuBaYQ6WWTcptADdkL3AIZ_jXJQHGgKag-Hm7BbQ,698
 nemo_evaluator_launcher/executors/lepton/deployment_helpers.py,sha256=AAIlHHn-WifevNosug0DlSDLN6NtjkclEu5LHyu1xq8,21799
-nemo_evaluator_launcher/executors/lepton/executor.py,sha256=P4xXS8pS8zgl_3TXjZKth2D5rTQ8TU3BmUiRD0uGiFo,35081
+nemo_evaluator_launcher/executors/lepton/executor.py,sha256=BeKZFflrooh_gbGoujY-cKOkarS-_VI0AoER91t5zvA,40240
 nemo_evaluator_launcher/executors/lepton/job_helpers.py,sha256=6baTxcygfP1oFgAJ7I9EL4xRlcJDWqbqzZoE1CRrwSk,13528
 nemo_evaluator_launcher/executors/local/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/local/executor.py,sha256=1OR74MI-raqqe-AJXUx7hybe5h-HCokye6ru6V6T1ak,20495
-nemo_evaluator_launcher/executors/local/run.template.sh,sha256=fk1AeJYw0s7ucb6aBnoBAMW0tckEgyIKk9ErLm1DYh0,3833
+nemo_evaluator_launcher/executors/local/executor.py,sha256=OUJ4B8qvE_mz9tXTYkwDqEjtOLtJzzbvOHDW35LYR4U,21431
+nemo_evaluator_launcher/executors/local/run.template.sh,sha256=uPofo2g8f7LsS0uL0YK0Y5YFFCQ5fAeiAkYMGBTalpg,3966
 nemo_evaluator_launcher/executors/slurm/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/slurm/executor.py,sha256=5pxq4kf85mrgwsO4-qH3I3Nq_H0qKsw3_wqFzILXy6c,39308
+nemo_evaluator_launcher/executors/slurm/executor.py,sha256=J_0sYzCcbcGzitf_oC9ncguoD2eG-K3osJxmZTW-9tE,40096
 nemo_evaluator_launcher/exporters/__init__.py,sha256=mBXG9FG48FeYrs8sF0zA2mgo1eqBmRgoml7zjJrqDso,1323
 nemo_evaluator_launcher/exporters/base.py,sha256=0BEqS-Zjez-KsrGE9yfo8S5w2uwMW3btBZve3SiiUp0,4307
 nemo_evaluator_launcher/exporters/gsheets.py,sha256=hBOL3vaomCW2fPMDEOQWkZkFCgF4jCoS4U5ZlsNVENs,15911
@@ -51,9 +52,9 @@ nemo_evaluator_launcher/exporters/registry.py,sha256=XsPTv_SBAFjcErO6BJ3OHqs3EvX
 nemo_evaluator_launcher/exporters/utils.py,sha256=XZVgTDmoa20tjEMwez0oUSpYpjt3ILV75D4KWuHtZ80,23119
 nemo_evaluator_launcher/exporters/wandb.py,sha256=1qRUV_YE1Ury7rH7KH65AabR7gmEQ38kXBh2XrfiEpE,18082
 nemo_evaluator_launcher/resources/mapping.toml,sha256=uOg4Y-gDXXskbbba2vuwJ5FLJ3W0kSZz7Fap_nJnFQc,11322
-nemo_evaluator_launcher-0.1.16.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.16.dist-info/METADATA,sha256=j3lLdhIOEMh4PaSAwb-wdtLrFheZidOcXzIiFTxCX_U,28725
-nemo_evaluator_launcher-0.1.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.16.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
-nemo_evaluator_launcher-0.1.16.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.16.dist-info/RECORD,,
+nemo_evaluator_launcher-0.1.18.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.18.dist-info/METADATA,sha256=QnzvHFzKYVGHhjKGNp55ErRQbG_FIscLpOyAhwgTS4U,28725
+nemo_evaluator_launcher-0.1.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.18.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
+nemo_evaluator_launcher-0.1.18.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.18.dist-info/RECORD,,