nemo-evaluator-launcher 0.1.8__tar.gz → 0.1.10__tar.gz

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.8
+Version: 0.1.10
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/api/functional.py

@@ -99,11 +99,13 @@ def run_eval(cfg: RunConfig, dry_run: bool = False) -> Optional[str]:
     return get_executor(cfg.execution.type).execute_eval(cfg, dry_run)
 
 
-def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
+def get_status(ids_or_prefixes: list[str]) -> list[dict[str, Any]]:
     """Get status of jobs by their IDs or invocation IDs.
 
     Args:
-        job_ids: List of job IDs or invocation IDs to check status for.
+        job_ids: List of job IDs or invocation IDs to check status for. Short ones are allowed,
+                 we would try to match the full ones from prefixes if no collisions are
+                 present.
 
     Returns:
         list[dict[str, Any]]: List of status dictionaries for each job or invocation.
@@ -114,14 +116,14 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
     db = ExecutionDB()
     results: List[dict[str, Any]] = []
 
-    for job_id in job_ids:
-        # If id looks like an invocation_id (8 hex digits, no dot), get all jobs for it
-        if len(job_id) == 8 and "." not in job_id:
-            jobs = db.get_jobs(job_id)
+    for id_or_prefix in ids_or_prefixes:
+        # If id looks like an invocation_id (no dot), get all jobs for it
+        if "." not in id_or_prefix:
+            jobs = db.get_jobs(id_or_prefix)
             if not jobs:
                 results.append(
                     {
-                        "invocation": job_id,
+                        "invocation": id_or_prefix,
                         "job_id": None,
                         "status": "not_found",
                         "data": {},
@@ -136,7 +138,7 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
             except ValueError as e:
                 results.append(
                     {
-                        "invocation": job_id,
+                        "invocation": id_or_prefix,
                         "job_id": None,
                         "status": "error",
                         "data": {"error": str(e)},
@@ -146,7 +148,7 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
 
             # Get status from the executor for all jobs in the invocation
             try:
-                status_list = executor_cls.get_status(job_id)
+                status_list = executor_cls.get_status(id_or_prefix)
 
                 # Create a result for each job in the invocation
                 for job_id_in_invocation, job_data in jobs.items():
@@ -161,7 +163,7 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
 
                     results.append(
                         {
-                            "invocation": job_id,
+                            "invocation": job_data.invocation_id,
                             "job_id": job_id_in_invocation,
                             "status": (
                                 job_status if job_status is not None else "unknown"
@@ -176,7 +178,7 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
             except Exception as e:
                 results.append(
                     {
-                        "invocation": job_id,
+                        "invocation": id_or_prefix,
                         "job_id": None,
                         "status": "error",
                         "data": {"error": str(e)},
@@ -184,13 +186,13 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
                 )
         else:
             # Otherwise, treat as job_id
-            single_job_data: Optional[JobData] = db.get_job(job_id)
+            single_job_data: Optional[JobData] = db.get_job(id_or_prefix)
 
             if single_job_data is None:
                 results.append(
                     {
                         "invocation": None,
-                        "job_id": job_id,
+                        "job_id": id_or_prefix,
                         "status": "not_found",
                         "data": {},
                     }
@@ -204,7 +206,7 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
                 results.append(
                     {
                         "invocation": None,
-                        "job_id": job_id,
+                        "job_id": id_or_prefix,
                         "status": "error",
                         "data": {"error": str(e)},
                     }
@@ -213,13 +215,13 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
 
             # Get status from the executor
             try:
-                status_list = executor_cls.get_status(job_id)
+                status_list = executor_cls.get_status(id_or_prefix)
 
                 if not status_list:
                     results.append(
                         {
                             "invocation": single_job_data.invocation_id,
-                            "job_id": job_id,
+                            "job_id": single_job_data.job_id,
                             "status": "unknown",
                             "data": single_job_data.data,
                         }
@@ -229,7 +231,7 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
                     results.append(
                         {
                             "invocation": single_job_data.invocation_id,
-                            "job_id": job_id,
+                            "job_id": single_job_data.job_id,
                             "status": (
                                 status_list[0].state.value if status_list else "unknown"
                             ),
@@ -246,7 +248,9 @@ def get_status(job_ids: list[str]) -> list[dict[str, Any]]:
                         "invocation": (
                             single_job_data.invocation_id if single_job_data else None
                         ),
-                        "job_id": job_id,
+                        "job_id": (
+                            single_job_data.job_id if single_job_data else id_or_prefix
+                        ),
                         "status": "error",
                         "data": {"error": str(e)},
                     }

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/cli/main.py

@@ -15,6 +15,8 @@
 #
 """Main CLI module using simple-parsing with subcommands."""
 
+import os
+
 from simple_parsing import ArgumentParser
 
 import nemo_evaluator_launcher.cli.export as export
@@ -29,6 +31,22 @@ from nemo_evaluator_launcher.common.logging_utils import logger
 VERSION_HELP = "Show version information"
 
 
+def is_verbose_enabled(args) -> bool:
+    """Check if verbose flag is enabled in any subcommand."""
+    # Check global verbose flag
+    if hasattr(args, "verbose") and args.verbose:
+        return True
+
+    # Check subcommand verbose flags
+    subcommands = ["run", "status", "kill", "tasks_alias", "tasks", "runs", "export"]
+    for subcmd in subcommands:
+        if hasattr(args, subcmd) and hasattr(getattr(args, subcmd), "verbose"):
+            if getattr(getattr(args, subcmd), "verbose"):
+                return True
+
+    return False
+
+
 def create_parser() -> ArgumentParser:
     """Create and configure the CLI argument parser with subcommands."""
     parser = ArgumentParser()
@@ -36,6 +54,14 @@ def create_parser() -> ArgumentParser:
     # Add --version flag at the top level
     parser.add_argument("--version", action="store_true", help=VERSION_HELP)
 
+    # Add --verbose/-v flag for debug logging
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging (sets LOG_LEVEL=DEBUG)",
+    )
+
     subparsers = parser.add_subparsers(dest="command", required=False)
 
     # Version subcommand
@@ -50,12 +76,24 @@ def create_parser() -> ArgumentParser:
     run_parser = subparsers.add_parser(
         "run", help="Run evaluation", description="Run evaluation"
     )
+    run_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging (sets LOG_LEVEL=DEBUG)",
+    )
     run_parser.add_arguments(run.Cmd, dest="run")
 
     # Status subcommand
     status_parser = subparsers.add_parser(
         "status", help="Check job status", description="Check job status"
     )
+    status_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging (sets LOG_LEVEL=DEBUG)",
+    )
     status_parser.add_arguments(status.Cmd, dest="status")
 
     # Kill subcommand
@@ -64,12 +102,24 @@ def create_parser() -> ArgumentParser:
         help="Kill a job or invocation",
         description="Kill a job (e.g., aefc4819.0) or entire invocation (e.g., aefc4819) by its ID",
     )
+    kill_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging (sets LOG_LEVEL=DEBUG)",
+    )
     kill_parser.add_arguments(kill.Cmd, dest="kill")
 
     # Ls subcommand (with nested subcommands)
     ls_parser = subparsers.add_parser(
         "ls", help="List resources", description="List tasks or runs"
     )
+    ls_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging (sets LOG_LEVEL=DEBUG)",
+    )
     # Add arguments from `ls tasks` so that they work with `ls` as default alias
     ls_parser.add_arguments(ls_tasks.Cmd, dest="tasks_alias")
 
@@ -95,6 +145,12 @@ def create_parser() -> ArgumentParser:
         help="Export evaluation results",
         description="Export evaluation results takes a List of invocation ids and a list of destinations(local, gitlab, wandb)",
     )
+    export_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging (sets LOG_LEVEL=DEBUG)",
+    )
     export_parser.add_arguments(export.ExportCmd, dest="export")
 
     return parser
@@ -105,6 +161,10 @@ def main() -> None:
     parser = create_parser()
     args = parser.parse_args()
 
+    # Handle --verbose flag
+    if is_verbose_enabled(args):
+        os.environ["LOG_LEVEL"] = "DEBUG"
+
     # Handle --version flag
     if hasattr(args, "version") and args.version:
         version_cmd = version.Cmd()

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/cli/run.py

@@ -59,6 +59,13 @@ class Cmd:
         alias=["-n", "--dry-run"],
         metadata={"help": "Do not run the evaluation, just print the config."},
     )
+    config_output: str | None = field(
+        default=None,
+        alias=["--config-output"],
+        metadata={
+            "help": "Directory to save the complete run config. Defaults to ~/.nemo-evaluator/run_configs/"
+        },
+    )
 
     def execute(self) -> None:
         # Import heavy dependencies only when needed
@@ -93,12 +100,19 @@ class Cmd:
 
         invocation_id = run_eval(config, self.dry_run)
 
-        # Save the complete configuration to the raw_configs directory
+        # Save the complete configuration
         if not self.dry_run and invocation_id is not None:
-            # Create ~/.nemo-evaluator/run_configs directory
-            home_dir = pathlib.Path.home()
-            run_configs_dir = home_dir / ".nemo-evaluator" / "run_configs"
-            run_configs_dir.mkdir(parents=True, exist_ok=True)
+            # Determine config output directory
+            if self.config_output:
+                # Use custom directory specified by --config-output
+                config_dir = pathlib.Path(self.config_output)
+            else:
+                # Default to original location: ~/.nemo-evaluator/run_configs
+                home_dir = pathlib.Path.home()
+                config_dir = home_dir / ".nemo-evaluator" / "run_configs"
+
+            # Ensure the directory exists
+            config_dir.mkdir(parents=True, exist_ok=True)
 
             # Convert DictConfig to dict and save as YAML
             config_dict = OmegaConf.to_container(config, resolve=True)
@@ -108,7 +122,7 @@ class Cmd:
 
             # Create config filename with invocation ID
             config_filename = f"{invocation_id}_config.yml"
-            config_path = run_configs_dir / config_filename
+            config_path = config_dir / config_filename
 
             # Save the complete Hydra configuration
             with open(config_path, "w") as f:

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/cli/status.py

@@ -17,6 +17,8 @@ from dataclasses import dataclass
 
 from simple_parsing import field
 
+from nemo_evaluator_launcher.executors.base import ExecutionState
+
 
 @dataclass
 class Cmd:
@@ -96,10 +98,14 @@ class Cmd:
             else:
                 location = ""
 
+            # Format status with visual indicators and colors
+            status = job.get("status", "")
+            formatted_status = self._format_status_with_indicators(status)
+
             rows.append(
                 [
                     job.get("job_id", ""),
-                    job.get("status", ""),
+                    formatted_status,
                     # job.get("progress", ""), temporarily disabled as this is a WIP feature
                     executor_info,
                     location,
@@ -108,7 +114,10 @@ class Cmd:
 
         # Calculate column widths and print
         widths = [
-            max(len(str(headers[i])), max(len(str(row[i])) for row in rows))
+            max(
+                len(str(headers[i])),
+                max(len(self._strip_ansi_codes(str(row[i]))) for row in rows),
+            )
             for i in range(len(headers))
         ]
 
@@ -119,4 +128,34 @@ class Cmd:
         print("-" * len(header_row))
 
         for row in rows:
-            print(" | ".join(str(row[i]).ljust(widths[i]) for i in range(len(row))))
+            # Adjust padding for ANSI color codes
+            formatted_row = []
+            for i in range(len(row)):
+                content = str(row[i])
+                visible_length = len(self._strip_ansi_codes(content))
+                padding = widths[i] - visible_length
+                formatted_row.append(content + " " * padding)
+            print(" | ".join(formatted_row))
+
+    def _format_status_with_indicators(self, status: str) -> str:
+        """Format status with Unicode visual indicators only."""
+        # Status mapping based on ExecutionState enum
+        status_formats = {
+            ExecutionState.SUCCESS.value: "\033[32m✓ SUCCESS\033[0m",  # Green Unicode checkmark
+            ExecutionState.FAILED.value: "\033[31m✗ FAILED\033[0m",  # Red Unicode X
+            ExecutionState.RUNNING.value: "\033[33m▶ RUNNING\033[0m",  # Yellow Unicode play button
+            ExecutionState.PENDING.value: "\033[36m⏳ PENDING\033[0m",  # Cyan Unicode hourglass
+            ExecutionState.KILLED.value: "\033[35m✗ KILLED\033[0m",  # Magenta Unicode X
+            # Additional states for error handling
+            "not_found": "\033[90m? NOT FOUND\033[0m",  # Gray question mark
+            "error": "\033[31m✗ ERROR\033[0m",  # Red Unicode X
+        }
+
+        return status_formats.get(status.lower(), f"\033[90m? {status.upper()}\033[0m")
+
+    def _strip_ansi_codes(self, text: str) -> str:
+        """Remove ANSI color codes from text for length calculation."""
+        import re
+
+        ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+        return ansi_escape.sub("", text)

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/common/execdb.py

@@ -29,15 +29,15 @@ EXEC_DB_FILE = EXEC_DB_DIR / "exec.v1.jsonl"
 
 
 def generate_invocation_id() -> str:
-    """Generate a unique invocation ID as an 8-digit hex string."""
-    return secrets.token_hex(4)
+    """Generate a unique invocation ID as an 16-digit hex string."""
+    return secrets.token_hex(8)
 
 
 def generate_job_id(invocation_id: str, index: int) -> str:
     """Generate a job ID as <invocation_id>.<n>.
 
     Args:
-        invocation_id: The invocation group ID (8-digit hex).
+        invocation_id: The invocation group ID (16-digit hex).
         index: The job index (0-based integer).
     Returns:
         The job ID string.
@@ -50,7 +50,7 @@ class JobData:
     """Data structure for job execution information.
 
     Attributes:
-        invocation_id: 8-digit hex string.
+        invocation_id: 16-digit hex string.
         job_id: <invocation_id>.<n> string.
         timestamp: Unix timestamp when the job was created.
         executor: Name of the executor that handled this job.
@@ -148,41 +148,135 @@ class ExecutionDB:
             )
             raise
 
+    def _resolve_invocation_id(self, short_id: str) -> Optional[str]:
+        """Resolve a short invocation ID to the full one.
+
+        Args:
+            short_id: Partial or full invocation ID.
+
+        Returns:
+            Full invocation ID if found uniquely, None if not found.
+
+        Raises:
+            ValueError: If the short_id matches multiple invocation IDs.
+        """
+        if not short_id:
+            return None
+
+        short_id = short_id.lower()
+
+        # NOTE(agronskiy): this is a non-optimized implementation that assumes small amount
+        # of jobs in ExecDB(), a typical scenario. Speeding up would involve building a
+        # prefix tree when loading invocations/jobs.
+        matches = [
+            inv_id
+            for inv_id in self._invocations.keys()
+            if inv_id.lower().startswith(short_id)
+        ]
+
+        if len(matches) == 1:
+            return matches[0]
+        elif len(matches) > 1:
+            raise ValueError(f"Ambiguous invocation ID '{short_id}': matches {matches}")
+        else:
+            return None
+
+    def _resolve_job_id(self, short_job_id: str) -> Optional[str]:
+        """Resolve a short job ID to the full one.
+
+        Args:
+            short_job_id: Partial or full job ID.
+
+        Returns:
+            Full job ID if found uniquely, None if not found.
+
+        Raises:
+            ValueError: If the short_job_id matches multiple job IDs.
+        """
+        if not short_job_id:
+            return None
+
+        # Normalize to lowercase for case-insensitive matching
+        short_job_id = short_job_id.lower()
+
+        if "." in short_job_id:
+            parts = short_job_id.split(".", 1)
+            short_inv_id, job_index = parts[0], parts[1]
+
+            # Resolve the invocation part
+            full_inv_id = self._resolve_invocation_id(short_inv_id)
+            if full_inv_id:
+                candidate_job_id = f"{full_inv_id}.{job_index}"
+                if candidate_job_id in self._jobs:
+                    return candidate_job_id
+
+        # NOTE(agronskiy): unfortunately, due to legacy, there exist usecases where
+        # job_id is the same format as invocation_id
+        candidate_job_id = self._resolve_invocation_id(short_job_id)
+        if candidate_job_id and candidate_job_id in self._jobs:
+            return candidate_job_id
+
+        return None
+
     def get_job(self, job_id: str) -> Optional[JobData]:
-        return self._jobs.get(job_id)
+        """Get job by full or partial job ID.
 
-    def get_jobs(self, invocation_id: str) -> Dict[str, JobData]:
-        job_ids = self._invocations.get(invocation_id, [])
-        return {
-            job_id: self._jobs[job_id] for job_id in job_ids if job_id in self._jobs
-        }
+        Args:
+            job_id: Full or partial job ID.
 
-    def get_invocation_jobs(self, invocation_id: str) -> List[str]:
-        return self._invocations.get(invocation_id, [])
+        Returns:
+            JobData if found, None otherwise.
 
-    def get_all_jobs(self) -> Dict[str, JobData]:
-        """Return a copy of all jobs in the execution DB."""
-        return dict(self._jobs)
+        Raises:
+            ValueError: If the job_id matches multiple jobs.
+        """
+        resolved_id = self._resolve_job_id(job_id)
+        if resolved_id:
+            return self._jobs.get(resolved_id)
 
+        return None
 
-def write_job(job: JobData) -> None:
-    db = ExecutionDB()
-    db.write_job(job)
+    def get_jobs(self, invocation_id: str) -> Dict[str, JobData]:
+        """Get all jobs for a full or partial invocation ID.
 
+        Args:
+            invocation_id: Full or partial invocation ID.
 
-def get_job(job_id: str) -> Optional[JobData]:
-    db = ExecutionDB()
-    return db.get_job(job_id)
+        Returns:
+            Dictionary mapping job_id to JobData for all jobs in the invocation.
 
+        Raises:
+            ValueError: If the invocation_id matches multiple invocations.
+        """
+        resolved_inv_id = self._resolve_invocation_id(invocation_id)
+        if not resolved_inv_id:
+            return {}
 
-def get_jobs(invocation_id: str) -> Dict[str, JobData]:
-    db = ExecutionDB()
-    return db.get_jobs(invocation_id)
+        job_ids = self._invocations.get(resolved_inv_id, [])
+        return {
+            job_id: self._jobs[job_id] for job_id in job_ids if job_id in self._jobs
+        }
+
+    def get_invocation_jobs(self, invocation_id: str) -> List[str]:
+        """Get job IDs for a full or partial invocation ID.
+
+        Args:
+            invocation_id: Full or partial invocation ID.
 
+        Returns:
+            List of job IDs for the invocation.
 
-def get_all_jobs() -> Dict[str, JobData]:
-    db = ExecutionDB()
-    return db.get_all_jobs()
+        Raises:
+            ValueError: If the invocation_id matches multiple invocations.
+        """
+        resolved_inv_id = self._resolve_invocation_id(invocation_id)
+        if not resolved_inv_id:
+            return []
+        return self._invocations.get(resolved_inv_id, [])
+
+    def get_all_jobs(self) -> Dict[str, JobData]:
+        """Return a copy of all jobs in the execution DB."""
+        return dict(self._jobs)
 
 
 # Ensure all the paths

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/common/helpers.py

@@ -75,7 +75,7 @@ def get_eval_factory_command(
     create_file_cmd = _yaml_to_echo_command(
         yaml.safe_dump(config_fields), "config_ef.yaml"
     )
-    eval_command = f"""eval-factory run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
+    eval_command = f"""cmd=$([[ $(command -v nemo-evaluator) ]] && echo 'nemo-evaluator' || echo 'eval-factory') && $cmd run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
 
     if overrides:
         eval_command = f"{eval_command} --overrides {overrides_str}"

nemo_evaluator_launcher-0.1.10/src/nemo_evaluator_launcher/configs/deployment/generic.yaml

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Generic server deployment configuration template
+# 
+type: generic
+image: ???  # Docker image to use for deployment
+command: ???  # Command to run the server
+
+# Server configuration
+port: 8000
+served_model_name: ???  # Name of the served model (used in command templates and evaluation)
+extra_args: ""  # Additional command line arguments
+env_vars: {}  # Environment variables as {name: value} dict
+checkpoint_path: null  # Path to model checkpoint
+
+# API endpoints (customize based on your server)
+endpoints:
+  chat: /v1/chat/completions
+  completions: /v1/completions
+  health: /health

nemo_evaluator_launcher-0.1.10/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml

@@ -0,0 +1,24 @@
+type: trtllm
+image: nvcr.io/nvidia/tensorrt-llm/release:1.0.0
+checkpoint_path: ???
+served_model_name: ???
+port: 8000
+tensor_parallel_size: 4
+pipeline_parallel_size: 1
+extra_args: ""
+
+endpoints:
+  chat: /v1/chat/completions
+  completions: /v1/completions
+  health: /health
+
+command:
+  mpirun --allow-run-as-root --oversubscribe
+  trtllm-serve serve /checkpoint
+  --tp_size=${deployment.tensor_parallel_size}
+  --pp_size=${deployment.pipeline_parallel_size}
+  --host 0.0.0.0
+  --port ${deployment.port}
+  --backend pytorch
+  --trust_remote_code
+  ${deployment.extra_args}

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/executors/lepton/executor.py

@@ -559,7 +559,7 @@ class LeptonExecutor(BaseExecutor):
         db = ExecutionDB()
 
         # If id looks like an invocation_id (8 hex digits, no dot), get all jobs for it
-        if len(id) == 8 and "." not in id:
+        if "." not in id:
             return _get_statuses_for_invocation_id(id=id, db=db)
         # Otherwise, treat as job_id
         job_data = db.get_job(id)

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/executors/local/executor.py

@@ -185,26 +185,6 @@ class LocalExecutor(BaseExecutor):
             run_all_sequentially_sh_content
         )
 
-        # Save launched jobs metadata
-        db = ExecutionDB()
-        for job_id, task, evaluation_task in zip(
-            job_ids, cfg.evaluation.tasks, evaluation_tasks
-        ):
-            db.write_job(
-                job=JobData(
-                    invocation_id=invocation_id,
-                    job_id=job_id,
-                    timestamp=time.time(),
-                    executor="local",
-                    data={
-                        "output_dir": str(evaluation_task["output_dir"]),
-                        "container": evaluation_task["container_name"],
-                        "eval_image": evaluation_task["eval_image"],
-                    },
-                    config=OmegaConf.to_object(cfg),
-                )
-            )
-
         if dry_run:
             print("\n\n=============================================\n\n")
             print(f"DRY RUN: Scripts prepared and saved to {output_dir}")
@@ -225,6 +205,26 @@ class LocalExecutor(BaseExecutor):
             print("\nTo execute, run without --dry-run")
             return invocation_id
 
+        # Save launched jobs metadata
+        db = ExecutionDB()
+        for job_id, task, evaluation_task in zip(
+            job_ids, cfg.evaluation.tasks, evaluation_tasks
+        ):
+            db.write_job(
+                job=JobData(
+                    invocation_id=invocation_id,
+                    job_id=job_id,
+                    timestamp=time.time(),
+                    executor="local",
+                    data={
+                        "output_dir": str(evaluation_task["output_dir"]),
+                        "container": evaluation_task["container_name"],
+                        "eval_image": evaluation_task["eval_image"],
+                    },
+                    config=OmegaConf.to_object(cfg),
+                )
+            )
+
         # Launch bash scripts with Popen for non-blocking execution.
         # To ensure subprocess continues after python exits:
         # - on Unix-like systems, to fully detach the subprocess
@@ -281,8 +281,8 @@ class LocalExecutor(BaseExecutor):
         """
         db = ExecutionDB()
 
-        # If id looks like an invocation_id (8 hex digits, no dot), get all jobs for it
-        if len(id) == 8 and "." not in id:
+        # If id looks like an invocation_id (no dot), get all jobs for it
+        if "." not in id:
             jobs = db.get_jobs(id)
             statuses: List[ExecutionStatus] = []
             for job_id, _ in jobs.items():

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/executors/local/run.template.sh

@@ -34,7 +34,7 @@ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.pre-start"
 # Docker run with eval factory command
 (
     echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.running"
-    docker run --rm --shm-size=100g \
+    docker run --rm --shm-size=100g --network=host \
       --name {{ task.container_name }} \
       --volume "$artifacts_dir":/results \
       {% for env_var in task.env_vars -%}

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/executors/slurm/executor.py

@@ -204,8 +204,8 @@ class SlurmExecutor(BaseExecutor):
         """
         db = ExecutionDB()
 
-        # If id looks like an invocation_id (8 hex digits, no dot), get all jobs for it
-        if len(id) == 8 and "." not in id:
+        # If id looks like an invocation_id (no dot), get all jobs for it
+        if "." not in id:
             jobs = db.get_jobs(id)
             if not jobs:
                 return []

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 8
+PATCH = 10
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.8
+Version: 0.1.10
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.8 → nemo_evaluator_launcher-0.1.10}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt

@@ -29,9 +29,11 @@ src/nemo_evaluator_launcher/common/logging_utils.py
 src/nemo_evaluator_launcher/common/mapping.py
 src/nemo_evaluator_launcher/configs/__init__.py
 src/nemo_evaluator_launcher/configs/default.yaml
+src/nemo_evaluator_launcher/configs/deployment/generic.yaml
 src/nemo_evaluator_launcher/configs/deployment/nim.yaml
 src/nemo_evaluator_launcher/configs/deployment/none.yaml
 src/nemo_evaluator_launcher/configs/deployment/sglang.yaml
+src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml
 src/nemo_evaluator_launcher/configs/deployment/vllm.yaml
 src/nemo_evaluator_launcher/configs/execution/local.yaml
 src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml