nemo-evaluator-launcher 0.1.56__py3-none-any.whl → 0.1.67__py3-none-any.whl

nemo_evaluator_launcher/api/types.py

@@ -18,7 +18,7 @@
 This module defines data structures and helpers for configuration and type safety in the API layer.
 """
 
-import os
+import pathlib
 import warnings
 from dataclasses import dataclass
 from typing import cast
@@ -42,33 +42,40 @@ from nemo_evaluator_launcher.common.logging_utils import logger
 class RunConfig(DictConfig):
     @staticmethod
     def from_hydra(
-        config_name: str = "default",
-        config_dir: str | None = None,
-        hydra_overrides: list[str] = [],
-        dict_overrides: dict = {},
+        config: str | None = None,
+        hydra_overrides: list[str] | None = None,
+        dict_overrides: dict | None = None,
     ) -> "RunConfig":
         """Load configuration from Hydra and merge with dictionary overrides.
 
         Args:
-            config_name: Name of the Hydra configuration to load.
+            config: Optional full path to a config file (e.g. /path/to/my_config.yaml).
+                    If omitted, loads the internal default config from
+                    `nemo_evaluator_launcher.configs`.
             hydra_overrides: List of Hydra command-line style overrides.
             dict_overrides: Dictionary of configuration overrides to merge.
-            config_dir: Optional path to user config directory. If provided, Hydra will
-                       search in this directory first, then fall back to internal configs.
 
         Returns:
             RunConfig: Merged configuration object.
         """
-        overrides = hydra_overrides.copy()
+        overrides = list(hydra_overrides or [])
+        dict_overrides = dict_overrides or {}
+
+        resolved_config_path: str | None = None
+        config_name = "default"
+
         # Check if a GlobalHydra instance is already initialized and clear it
         if GlobalHydra.instance().is_initialized():
             GlobalHydra.instance().clear()
 
-        if config_dir:
-            # Convert relative path to absolute path if needed
-            if not os.path.isabs(config_dir):
-                config_dir = os.path.abspath(config_dir)
+        if config:
+            config_path = pathlib.Path(config).expanduser()
+            if not config_path.is_absolute():
+                config_path = (pathlib.Path.cwd() / config_path).resolve()
+            resolved_config_path = str(config_path)
 
+            config_dir = str(config_path.parent)
+            config_name = str(config_path.stem)
             hydra.initialize_config_dir(
                 config_dir=config_dir,
                 version_base=None,
@@ -90,7 +97,7 @@ class RunConfig(DictConfig):
         logger.debug(
             "Loaded run config from hydra",
             config_name=config_name,
-            config_dir=config_dir,
+            config=resolved_config_path,
             overrides=hydra_overrides,
             dict_overrides=dict_overrides,
             result=cfg,

nemo_evaluator_launcher/cli/run.py

@@ -27,7 +27,6 @@ from nemo_evaluator_launcher.common.printing_utils import (
     green,
     magenta,
     red,
-    yellow,
 )
 
 
@@ -42,20 +41,6 @@ class Cmd:
             "help": "Full path to config file. Uses Hydra by default (--config-mode=hydra). Use --config-mode=raw to load directly (bypasses Hydra)."
         },
     )
-    config_name: str = field(
-        default="default",
-        alias=["-c", "--config-name"],
-        metadata={
-            "help": "Config name to use. Consult `nemo_evaluator_launcher.configs`"
-        },
-    )
-    config_dir: str | None = field(
-        default=None,
-        alias=["-d", "--config-dir"],
-        metadata={
-            "help": "Path to user config directory. If provided, searches here first, then falls back to internal configs."
-        },
-    )
     config_mode: Literal["hydra", "raw"] = field(
         default="hydra",
         alias=["--config-mode"],
@@ -138,14 +123,6 @@ class Cmd:
         # Load configuration either from Hydra or directly from a config file
         if self.config_mode == "raw" and self.config:
             # Validate that raw config loading is not used with other config options
-            if self.config_name != "default":
-                raise ValueError(
-                    "Cannot use --config-mode=raw with --config-name. Raw mode only works with --config."
-                )
-            if self.config_dir is not None:
-                raise ValueError(
-                    "Cannot use --config-mode=raw with --config-dir. Raw mode only works with --config."
-                )
             if self.override:
                 raise ValueError(
                     "Cannot use --config-mode=raw with --override. Raw mode only works with --config."
@@ -158,23 +135,9 @@ class Cmd:
             # Create RunConfig from the loaded data
             config = OmegaConf.create(config_dict)
         else:
-            # Handle --config parameter: split path into config_dir and config_name for Hydra
-            if self.config:
-                if self.config_name != "default":
-                    raise ValueError("Cannot use --config with --config-name")
-                if self.config_dir is not None:
-                    raise ValueError("Cannot use --config with --config-dir")
-                config_path = pathlib.Path(self.config)
-                config_dir = str(config_path.parent)
-                config_name = str(config_path.stem)
-            else:
-                config_dir = self.config_dir
-                config_name = self.config_name
-
             # Load the complete Hydra configuration
             config = RunConfig.from_hydra(
-                config_dir=config_dir,
-                config_name=config_name,
+                config=self.config,
                 hydra_overrides=self.override,
             )
 
@@ -283,16 +246,4 @@ class Cmd:
                 )
             )
 
-        # Warn if both config_dir and config_name are provided (and config_name is not default)
-        if (
-            self.config is None
-            and self.config_dir is not None
-            and self.config_name != "default"
-        ):
-            joint_path = pathlib.Path(self.config_dir) / f"{self.config_name}.yaml"
-            print(
-                yellow(
-                    f"Warning: Using --config-dir and --config-name together is deprecated. "
-                    f"Please use --config {joint_path} instead."
-                )
-            )
+        # Done.

nemo_evaluator_launcher/common/mapping.py

@@ -16,8 +16,6 @@
 import pathlib
 from typing import Any
 
-import yaml
-
 from nemo_evaluator_launcher.common.container_metadata import (
     TaskIntermediateRepresentation,
     load_tasks_from_tasks_file,
@@ -25,201 +23,6 @@ from nemo_evaluator_launcher.common.container_metadata import (
 from nemo_evaluator_launcher.common.logging_utils import logger
 
 
-def _load_packaged_resource(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
-    """Deprecated: mapping.toml support was removed in favor of packaged IRs."""
-    raise RuntimeError(
-        "mapping.toml is no longer supported. Use packaged IRs (all_tasks_irs.yaml) instead."
-    )
-
-
-def _process_mapping(mapping_toml: dict) -> dict:
-    """Process the raw mapping TOML into the expected format.
-
-    Args:
-        mapping_toml: Raw mapping TOML data.
-    Returns:
-        dict: Processed mapping in the expected format.
-    """
-    mapping = {}
-    for harness_name, harness_data in mapping_toml.items():
-        # Skip entries that don't have the expected structure
-        if not isinstance(harness_data, dict):
-            logger.warning(
-                "Skipping invalid harness entry",
-                harness_name=harness_name,
-                reason="harness_data is not a dict",
-            )
-            continue
-
-        # Check if tasks field exists
-        if "tasks" not in harness_data:
-            logger.warning(
-                "Skipping harness entry without tasks",
-                harness_name=harness_name,
-            )
-            continue
-
-        if not isinstance(harness_data["tasks"], dict):
-            logger.warning(
-                "Skipping invalid harness entry",
-                harness_name=harness_name,
-                reason="tasks is not a dict",
-            )
-            continue
-
-        # Get container, which may be optional
-        container = harness_data.get("container")
-        if not container:
-            logger.debug(
-                "Harness entry without container",
-                harness_name=harness_name,
-            )
-
-        for endpoint_type, harness_tasks in harness_data["tasks"].items():
-            if not isinstance(harness_tasks, dict):
-                logger.warning(
-                    "Skipping invalid endpoint type",
-                    harness_name=harness_name,
-                    endpoint_type=endpoint_type,
-                    reason="harness_tasks is not a dict",
-                )
-                continue
-
-            for task_name, task_data in harness_tasks.items():
-                if not isinstance(task_data, dict):
-                    logger.warning(
-                        "Skipping invalid task entry",
-                        harness_name=harness_name,
-                        task_name=task_name,
-                        reason="task_data is not a dict",
-                    )
-                    continue
-
-                key = (harness_name, task_name)
-                if key in mapping:
-                    raise KeyError(
-                        f"(harness,task)-tuple key {repr(key)} already exists in the mapping"
-                    )
-
-                # Validate required fields exist in task_data
-                # task_name and harness_name are already validated above
-                # endpoint_type is validated as a key in harness_tasks
-                # task_data must be a dict (validated above)
-
-                mapping[key] = {
-                    "task": task_name,
-                    "harness": harness_name,
-                    "endpoint_type": endpoint_type,
-                }
-                # Only add container if it exists
-                if container:
-                    mapping[key]["container"] = container
-
-                # Validate task_data keys before updating
-                for task_data_key in task_data.keys():
-                    if task_data_key in mapping[key]:
-                        raise KeyError(
-                            f"{repr(task_data_key)} is not allowed as key under {repr(key)} in the mapping"
-                        )
-                    # Validate that task_data values are valid types (basic check)
-                    if task_data_key not in ("description", "type") and not isinstance(
-                        task_data[task_data_key],
-                        (str, int, float, bool, dict, list, type(None)),
-                    ):
-                        logger.warning(
-                            "Unexpected value type in task_data",
-                            harness_name=harness_name,
-                            task_name=task_name,
-                            key=task_data_key,
-                            value_type=type(task_data[task_data_key]).__name__,
-                        )
-
-                mapping[key].update(task_data)
-    return mapping
-
-
-def _extract_tasks_from_framework_yml(
-    framework_yml_content: str, harness_name: str, container: str
-) -> dict[tuple[str, str], dict]:
-    """Extract tasks from framework.yml content and return as mapping entries.
-
-    Args:
-        framework_yml_content: YAML content from framework.yml file
-        harness_name: Name of the harness
-        container: Container image string
-
-    Returns:
-        Dictionary mapping (harness_name, task_name) to task configuration
-    """
-    tasks = {}
-    try:
-        framework_data = yaml.safe_load(framework_yml_content)
-        if not framework_data or "evaluations" not in framework_data:
-            logger.warning(
-                "No evaluations found in framework.yml",
-                harness=harness_name,
-                container=container,
-            )
-            return tasks
-
-        evaluations = framework_data.get("evaluations", [])
-        for eval_config in evaluations:
-            task_name = eval_config.get("name")
-            description = eval_config.get("description", "")
-
-            if not task_name:
-                continue
-
-            # Extract endpoint types from the evaluation config
-            defaults = eval_config.get("defaults", {})
-            config = defaults.get("config", {})
-            supported_endpoint_types = config.get("supported_endpoint_types", ["chat"])
-            task_type = config.get("type", "")  # Extract type from defaults.config.type
-
-            # Use first endpoint type (mapping key is (harness, task), so one entry per task)
-            endpoint_type = (
-                supported_endpoint_types[0] if supported_endpoint_types else "chat"
-            )
-
-            key = (harness_name, task_name)
-            # Only add if not already in mapping (don't override existing entries)
-            if key not in tasks:
-                tasks[key] = {
-                    "task": task_name,
-                    "harness": harness_name,
-                    "container": container,
-                    "endpoint_type": endpoint_type,
-                    "description": description,
-                    "type": task_type,  # Store type from defaults.config.type
-                }
-                # Merge any additional config from defaults
-                if defaults:
-                    tasks[key].update(defaults)
-
-        logger.info(
-            "Extracted tasks from framework.yml",
-            harness=harness_name,
-            container=container,
-            num_tasks=len(tasks),
-        )
-    except yaml.YAMLError as e:
-        logger.warning(
-            "Failed to parse framework.yml",
-            harness=harness_name,
-            container=container,
-            error=str(e),
-        )
-    except Exception as e:
-        logger.warning(
-            "Error extracting tasks from framework.yml",
-            harness=harness_name,
-            container=container,
-            error=str(e),
-        )
-
-    return tasks
-
-
 def _convert_irs_to_mapping_format(
     tasks: list[TaskIntermediateRepresentation],
 ) -> dict[tuple[str, str], dict]:

nemo_evaluator_launcher/configs/deployment/nim.yaml

@@ -18,13 +18,15 @@ image: ??? # e.g., nvcr.io/nim/meta/llama-3.1-8b-instruct:1.8.6
 served_model_name: ???
 port: 8000
 
+command: /opt/nim/start_server.sh
+
 # NIM containers use default entrypoint - no custom command needed
 # Configuration is done via environment variables in lepton_config
 
 endpoints:
   chat: /v1/chat/completions
   completions: /v1/completions
-  health: /health
+  health: /v1/health/ready
 # Note: Environment variables should be configured in lepton_config.envs
 # Auto-derived environment variables from deployment config:
 # - SERVED_MODEL_NAME (from served_model_name)

nemo_evaluator_launcher/executors/slurm/executor.py

@@ -408,10 +408,10 @@ class SlurmExecutor(BaseExecutor):
             )
         statuses = []
         for i, slurm_job_id in enumerate(slurm_job_ids):
-            slurm_status = slurm_jobs_status[slurm_job_id]
+            slurm_status = slurm_jobs_status[slurm_job_id][0]
             if slurm_job_id in latest_slurm_job_ids:
                 latest_slurm_job_id = latest_slurm_job_ids[slurm_job_id]
-                slurm_status = latest_slurm_jobs_status[latest_slurm_job_id]
+                slurm_status = latest_slurm_jobs_status[latest_slurm_job_id][0]
             progress = progress_list[i]
             progress = progress if progress is not None else "unknown"
             execution_state = SlurmExecutor._map_slurm_state_to_execution_state(
@@ -644,7 +644,7 @@ def _create_slurm_sbatch_script(
         s += deployment_srun_cmd
 
         # wait for the server to initialize
-        health_path = cfg.deployment.get("health_check_path", "/health")
+        health_path = cfg.deployment.endpoints.get("health", "/health")
         # For multi-instance check all node IPs, for single instance check localhost
         if cfg.deployment.get("multiple_instances", False):
             ip_list = '"${NODES_IPS_ARRAY[@]}"'
@@ -710,7 +710,7 @@ def _create_slurm_sbatch_script(
 
     s += "# evaluation client\n"
     s += "srun --mpi pmix --overlap "
-    s += "--nodes 1 --ntasks 1 "  # Client always runs on single node
+    s += "--nodelist ${nodes_array[0]} --nodes 1 --ntasks 1 "
     s += "--container-image {} ".format(eval_image)
     evaluation_env_var_names = list(
         cfg.execution.get("env_vars", {}).get("evaluation", {})
@@ -835,7 +835,7 @@ def _generate_auto_export_section(
 
     s += "    # export\n"
     s += "    srun --mpi pmix --overlap "
-    s += "--nodes 1 --ntasks 1 "  # Client always runs on single node
+    s += "--nodelist ${nodes_array[0]} --nodes 1 --ntasks 1 "
     s += "--container-image {} ".format(export_image)
     if export_env:
         s += "--container-env {} ".format(",".join(export_env))
@@ -1009,8 +1009,55 @@ def _query_slurm_jobs_status(
     username: str,
     hostname: str,
     socket: str | None,
-) -> Dict[str, str]:
-    """Query SLURM for job statuses using sacct command.
+) -> Dict[str, tuple[str, str]]:
+    """Query SLURM for job statuses using squeue (for active jobs) and sacct (fallback).
+
+    This function first tries squeue which is more accurate for currently running jobs,
+    then falls back to sacct for completed/historical jobs that squeue doesn't show.
+    It also finds follow-up jobs (from autoresume) that depend on our known jobs.
+
+    Args:
+        slurm_job_ids: List of SLURM job IDs to query.
+        username: SSH username.
+        hostname: SSH hostname.
+        socket: control socket location or None
+
+    Returns:
+        Dict mapping from slurm_job_id to tuple of status, current_job_id.
+    """
+    if len(slurm_job_ids) == 0:
+        return {}
+
+    # First, try squeue for active jobs (more accurate for running jobs)
+    squeue_statuses = _query_squeue_for_jobs(slurm_job_ids, username, hostname, socket)
+
+    # For jobs not found in squeue, fall back to sacct
+    missing_jobs = [job_id for job_id in slurm_job_ids if job_id not in squeue_statuses]
+    sacct_statuses = {}
+
+    if missing_jobs:
+        sacct_statuses = _query_sacct_for_jobs(missing_jobs, username, hostname, socket)
+
+    # Combine results, preferring squeue data
+    combined_statuses = {**sacct_statuses, **squeue_statuses}
+
+    return combined_statuses
+
+
+def _query_squeue_for_jobs(
+    slurm_job_ids: List[str],
+    username: str,
+    hostname: str,
+    socket: str | None,
+) -> Dict[str, tuple[str, str]]:
+    """Query SLURM for active job statuses using squeue command.
+
+    This function finds:
+    1. Jobs that directly match our known job IDs
+    2. Follow-up jobs that depend on our known job IDs (from autoresume mechanism)
+
+    For follow-up jobs, returns the status mapped to the original job ID, along with
+    the actual current SLURM job ID.
 
     Args:
         slurm_job_ids: List of SLURM job IDs to query.
@@ -1019,10 +1066,77 @@ def _query_slurm_jobs_status(
         socket: control socket location or None
 
     Returns:
-        Dict mapping from slurm_job_id to returned slurm status.
+        Dict mapping from original slurm_job_id to tuple of status, current_job_id.
     """
     if len(slurm_job_ids) == 0:
         return {}
+
+    # Use squeue to get active jobs - more accurate than sacct for running jobs
+    squeue_command = "squeue -u {} -h -o '%i|%T|%E'".format(username)
+
+    ssh_command = ["ssh"]
+    if socket is not None:
+        ssh_command.append(f"-S {socket}")
+    ssh_command.append(f"{username}@{hostname}")
+    ssh_command.append(squeue_command)
+    ssh_command = " ".join(ssh_command)
+
+    completed_process = subprocess.run(
+        args=shlex.split(ssh_command),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    squeue_statuses = {}
+    dependent_jobs = []
+    if completed_process.returncode == 0:
+        squeue_output = completed_process.stdout.decode("utf-8")
+        squeue_output_lines = squeue_output.strip().split("\n")
+
+        for line in squeue_output_lines:
+            if not line.strip():
+                continue
+            parts = line.split("|")
+            if len(parts) >= 3:
+                job_id = parts[0].strip()
+                status = parts[1].strip()
+                dependency = parts[2].strip()
+                # Extract base job ID (handle array jobs like 123456_0 -> 123456)
+                base_job_id = job_id.split("_")[0].split("[")[0]
+                if base_job_id in slurm_job_ids:
+                    squeue_statuses[base_job_id] = status, base_job_id
+                elif dependency and dependency != "(null)":
+                    dependent_jobs.append((base_job_id, status, dependency))
+
+        for dep_job_id, dep_status, dependency in dependent_jobs:
+            for known_job_id in slurm_job_ids:
+                if known_job_id in dependency and known_job_id not in squeue_statuses:
+                    squeue_statuses[known_job_id] = dep_status, dep_job_id
+                    break
+
+    return squeue_statuses
+
+
+def _query_sacct_for_jobs(
+    slurm_job_ids: List[str],
+    username: str,
+    hostname: str,
+    socket: str | None,
+) -> Dict[str, tuple[str, str]]:
+    """Query SLURM for job statuses using sacct command (for completed/historical jobs).
+
+    Args:
+        slurm_job_ids: List of SLURM job IDs to query.
+        username: SSH username.
+        hostname: SSH hostname.
+        socket: control socket location or None
+
+    Returns:
+        Dict mapping from slurm_job_id to tuple of status, job_id.
+    """
+    if len(slurm_job_ids) == 0:
+        return {}
+
     sacct_command = "sacct -j {} --format='JobID,State%32' --noheader -P".format(
         ",".join(slurm_job_ids)
     )
@@ -1049,7 +1163,7 @@ def _query_slurm_jobs_status(
     slurm_jobs_status = {}
     for slurm_job_id in slurm_job_ids:
         slurm_job_status = _parse_slurm_job_status(slurm_job_id, sacct_output_lines)
-        slurm_jobs_status[slurm_job_id] = slurm_job_status
+        slurm_jobs_status[slurm_job_id] = slurm_job_status, slurm_job_id
     return slurm_jobs_status
 
 
@@ -1264,9 +1378,11 @@ def _generate_haproxy_config_with_placeholders(cfg):
     for i in range(num_nodes):
         nodes.append({"ip": f"{{IP_{i}}}", "port": cfg.deployment.port})
 
-    # Get health check parameters from execution config
+    # Get health check parameters - prefer proxy config, fallback to deployment.endpoints.health
     proxy_config = cfg.execution.get("proxy", {}).get("config", {})
-    health_check_path = proxy_config.get("health_check_path", "/health")
+    health_check_path = proxy_config.get(
+        "health_check_path", cfg.deployment.endpoints.get("health", "/health")
+    )
     health_check_status = proxy_config.get("health_check_status", 200)
     haproxy_port = proxy_config.get("haproxy_port", 5009)
 
@@ -1301,7 +1417,7 @@ def _generate_haproxy_config(cfg, nodes_ips):
         )
 
     # Get health check parameters from deployment config
-    health_check_path = cfg.deployment.get("health_check_path", "/health")
+    health_check_path = cfg.deployment.endpoints.get("health", "/health")
     health_check_status = cfg.deployment.get("health_check_status", 200)
     haproxy_port = cfg.deployment.get("haproxy_port", 5009)
 
@@ -1461,7 +1577,7 @@ def _generate_haproxy_srun_command(cfg, remote_task_subdir):
     s += "done\n"
     s += "\n"
     s += "srun --mpi pmix --overlap "
-    s += "--nodes 1 --ntasks 1 "
+    s += "--nodelist ${nodes_array[0]} --nodes 1 --ntasks 1 "
     s += f"--container-image {cfg.execution.get('proxy', {}).get('image', 'haproxy:latest')} "
     s += f"--container-mounts {remote_task_subdir}/proxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro "
     s += f"--output {remote_task_subdir}/logs/proxy-%A.log "

nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 56
+PATCH = 67
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator_launcher-0.1.56.dist-info → nemo_evaluator_launcher-0.1.67.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.56
+Version: 0.1.67
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.56.dist-info → nemo_evaluator_launcher-0.1.67.dist-info}/RECORD

@@ -1,8 +1,8 @@
 nemo_evaluator_launcher/__init__.py,sha256=GT38zGwbvBOSeU52WCRx-n9N49LvLGEV1PItgKC8orA,2320
-nemo_evaluator_launcher/package_info.py,sha256=zHKOOOIbhAWOrU7pN-acMaSYwI2zdqC5syv_zy4x21A,1586
+nemo_evaluator_launcher/package_info.py,sha256=HOfV93vn4SKitHYaUmZ6OXBBSz64WCDVCQPm6oliO6A,1586
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
 nemo_evaluator_launcher/api/functional.py,sha256=di5az6OkMSGBr5bWTT6JLgtqNBU8-fwAb_eZETyVulI,33641
-nemo_evaluator_launcher/api/types.py,sha256=W7ZQ9ZTPR6YxInxxsKE6NxuuQAg4pVYz6SRmFCFxY0A,3635
+nemo_evaluator_launcher/api/types.py,sha256=OVAI_mHYTqGc6IlXmCLRcQ1rbgwG7Oe4qpnbM0kCrfc,3884
 nemo_evaluator_launcher/api/utils.py,sha256=q5HArRj7PKgBfeH3bOX8q1U97yMyQQp72yRRA5JP9PE,818
 nemo_evaluator_launcher/cli/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/cli/export.py,sha256=GRXxusKDq_1qjMKN6MKOIjZ8x4u5ERgXwHSAGrvsGCY,11211
@@ -13,14 +13,14 @@ nemo_evaluator_launcher/cli/ls_runs.py,sha256=vJTwRdhVKLolnJuP8AnnQdJBE-BKZfCcCy
 nemo_evaluator_launcher/cli/ls_task.py,sha256=sChh9UQJK0Zh1g3RitVtm8o_6qJQuS75JGrHXf3Y0y4,10954
 nemo_evaluator_launcher/cli/ls_tasks.py,sha256=aMzuAiUUeTCeSN3XCtoneOm-BBSnOVfhgIzgxZ8Y8wI,11167
 nemo_evaluator_launcher/cli/main.py,sha256=3BwJXuB9WHmiTKu4FR2dyrVonwpo8YWcPfw3MSmqOu8,8148
-nemo_evaluator_launcher/cli/run.py,sha256=KGBIXiDfT5OUv09FaTl-GyOxDhjxWEIE1rGlUkccB2o,11336
+nemo_evaluator_launcher/cli/run.py,sha256=2ITc5ZIdwzmjZsQct7D0_SF8UccOb2xqzBQscj1vWEg,9210
 nemo_evaluator_launcher/cli/status.py,sha256=ANdu0JYnfKNvd1gXmdu_0FrbPG-g0A_R4leOuNXzenQ,5947
 nemo_evaluator_launcher/cli/version.py,sha256=GgMNTAd4S0bu3t-uVfVedAf7p6pymWDDwOaNm4WHOxQ,1998
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=WPzg5Iu2ojvFpBuYahSt3voP_iEUpoO8NgqMLUBwFxA,9767
 nemo_evaluator_launcher/common/helpers.py,sha256=pWYd1P5tbBSAK1JPv7dk_d_Sq92gJ4NIHXiiOEyNiZY,13218
 nemo_evaluator_launcher/common/logging_utils.py,sha256=7QkWlpA80QN5ipTUFJ198IiAsPRS36C6ISAAtNverbA,12338
-nemo_evaluator_launcher/common/mapping.py,sha256=O4Xg_0hgnQOOxAEGFlFJjHh_sTXvbIEUCgNXYxvfnDU,18035
+nemo_evaluator_launcher/common/mapping.py,sha256=cGjyjvPzOL6T1pshGCbaa1QLkOsqRRZJVcU0_XszY9M,10657
 nemo_evaluator_launcher/common/printing_utils.py,sha256=K9_-ENUIGHcL9i-fmGscfCJoV-uWJlaMKNToNx1Ypmg,2631
 nemo_evaluator_launcher/common/container_metadata/__init__.py,sha256=QxPpIUTWprJybsfsaNoHnpLNIGxIVERW0mAtZ-u-F3c,2145
 nemo_evaluator_launcher/common/container_metadata/intermediate_repr.py,sha256=aa9J0tdvdVYpGPu85vkdpBnFOpSReLdA0SNb45jAC0Y,17525
@@ -30,7 +30,7 @@ nemo_evaluator_launcher/common/container_metadata/utils.py,sha256=kaINUugS7Jhydm
 nemo_evaluator_launcher/configs/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/configs/default.yaml,sha256=JHFjSl3KByhggRMTTo9nesQATVoz18PJIV6KM5Wng64,974
 nemo_evaluator_launcher/configs/deployment/generic.yaml,sha256=8_Z0fcjZuH6GfV9jJkY_8CS18Tbsn0gV-nK1LfGr_Vg,1262
-nemo_evaluator_launcher/configs/deployment/nim.yaml,sha256=hRJGwCR2XIS3bUFWkXzmroLep81KaHa3Sn2_edfWbkU,1266
+nemo_evaluator_launcher/configs/deployment/nim.yaml,sha256=6YGGOyb-JIJqk2fOoo8NalUppbFaml7_lDBQXC9AVmw,1310
 nemo_evaluator_launcher/configs/deployment/none.yaml,sha256=buPada1yMz9ClPDbq63vPDzLGr_IubTLia91DG3i5Lo,684
 nemo_evaluator_launcher/configs/deployment/sglang.yaml,sha256=Yy2uOko7-HeJ1fE0LHK3AHUOAzaeVSfnDp6f7fRBA9s,1358
 nemo_evaluator_launcher/configs/deployment/trtllm.yaml,sha256=1-001_ylJOiQQiqhcIkX_8SFTkOGx47odB_4Myug_9A,568
@@ -49,7 +49,7 @@ nemo_evaluator_launcher/executors/local/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQn
 nemo_evaluator_launcher/executors/local/executor.py,sha256=lLBEv0UoB7PRKdVsYdcodW0klCn90t0ua3Sp41IZr4U,37012
 nemo_evaluator_launcher/executors/local/run.template.sh,sha256=S6qLqnqA3B4P_-ngklCU5dO9rKV0Qom5NWDwDHf5i0g,6103
 nemo_evaluator_launcher/executors/slurm/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/slurm/executor.py,sha256=zumFTbnKQzKNv5g7Y_PxsAXAsHIIvXKvIIOPLS8hoiA,59726
+nemo_evaluator_launcher/executors/slurm/executor.py,sha256=YtTNMKJX-6HJelZk4PI0XPqlVRJsHnOtEt_FszZ45t0,63919
 nemo_evaluator_launcher/executors/slurm/proxy.cfg.template,sha256=nPg_JyBHPcjI1RSxDvl-lBgcbakrbZLl4IAXNBc8LM0,594
 nemo_evaluator_launcher/exporters/__init__.py,sha256=mBXG9FG48FeYrs8sF0zA2mgo1eqBmRgoml7zjJrqDso,1323
 nemo_evaluator_launcher/exporters/base.py,sha256=0BEqS-Zjez-KsrGE9yfo8S5w2uwMW3btBZve3SiiUp0,4307
@@ -61,9 +61,9 @@ nemo_evaluator_launcher/exporters/utils.py,sha256=5HGBKoFTSCng-GhAJpvfAa7N52_r9U
 nemo_evaluator_launcher/exporters/wandb.py,sha256=1qRUV_YE1Ury7rH7KH65AabR7gmEQ38kXBh2XrfiEpE,18082
 nemo_evaluator_launcher/resources/all_tasks_irs.yaml,sha256=g2T5Jb_2zkJFZd4gqHmOLKiKR52ioz9adjOYcaSEnyE,978316
 nemo_evaluator_launcher/resources/mapping.toml,sha256=iB5tHeMYJMnnej4TwKmrmI1rq3zJ-rWymO8eWxvfGvk,3619
-nemo_evaluator_launcher-0.1.56.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.56.dist-info/METADATA,sha256=uKG_JgnSQcHFxj2Nxs-Qmq7jc39BCksOdQ-qMfevGUE,28760
-nemo_evaluator_launcher-0.1.56.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.56.dist-info/entry_points.txt,sha256=HPmLybw-y1y4NxjK7slFS4mtTB7p4pnjRvCtJLvSiZs,174
-nemo_evaluator_launcher-0.1.56.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.56.dist-info/RECORD,,
+nemo_evaluator_launcher-0.1.67.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.67.dist-info/METADATA,sha256=u7EwjsAfTzBcYcn9Ye8jQNPWpziL7ThNtJWz3CHNeLM,28760
+nemo_evaluator_launcher-0.1.67.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.67.dist-info/entry_points.txt,sha256=HPmLybw-y1y4NxjK7slFS4mtTB7p4pnjRvCtJLvSiZs,174
+nemo_evaluator_launcher-0.1.67.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.67.dist-info/RECORD,,