nemo-evaluator-launcher 0.1.0rc9__tar.gz → 0.1.2__tar.gz

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc9
+Version: 0.1.2
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/cli/export.py

@@ -20,8 +20,6 @@ from typing import Any, List, Optional
 
 from simple_parsing import field
 
-from nemo_evaluator_launcher.api.functional import export_results
-
 
 @dataclass
 class ExportCmd:
@@ -77,6 +75,9 @@ class ExportCmd:
 
     def execute(self) -> None:
         """Execute export."""
+        # Import heavy dependencies only when needed
+        from nemo_evaluator_launcher.api.functional import export_results
+
         config: dict[str, Any] = {
             "copy_logs": self.copy_logs,
             "only_required": self.only_required,

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/cli/kill.py

@@ -13,13 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import json
 from dataclasses import dataclass
 
 from simple_parsing import field
 
-from nemo_evaluator_launcher.api.functional import kill_job_or_invocation
-
 
 @dataclass
 class Cmd:
@@ -34,6 +31,11 @@ class Cmd:
 
     def execute(self) -> None:
         """Execute the kill command."""
+        # Import heavy dependencies only when needed
+        import json
+
+        from nemo_evaluator_launcher.api.functional import kill_job_or_invocation
+
         result = kill_job_or_invocation(self.id)
         # Output as JSON
         print(json.dumps(result, indent=2))

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/cli/ls_runs.py

@@ -20,11 +20,6 @@ from typing import Optional
 
 from simple_parsing import field
 
-from nemo_evaluator_launcher.api.functional import (
-    get_invocation_benchmarks,
-    list_all_invocations_summary,
-)
-
 
 @dataclass
 class Cmd:
@@ -41,6 +36,12 @@ class Cmd:
     )
 
     def execute(self) -> None:
+        # Import heavy dependencies only when needed
+        from nemo_evaluator_launcher.api.functional import (
+            get_invocation_benchmarks,
+            list_all_invocations_summary,
+        )
+
         rows = list_all_invocations_summary()
 
         if self.executor:

nemo_evaluator_launcher-0.1.2/src/nemo_evaluator_launcher/cli/ls_tasks.py

@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from collections import defaultdict
+from dataclasses import dataclass
+
+from simple_parsing import field
+
+
+@dataclass
+class Cmd:
+    """List command configuration."""
+
+    json: bool = field(
+        default=False,
+        action="store_true",
+        help="Print output as JSON instead of table format",
+    )
+
+    def execute(self) -> None:
+        # Import heavy dependencies only when needed
+        import json
+
+        from nemo_evaluator_launcher.api.functional import get_tasks_list
+
+        # TODO(dfridman): modify `get_tasks_list` to return a list of dicts in the first place
+        data = get_tasks_list()
+        headers = ["task", "endpoint_type", "harness", "container"]
+        supported_benchmarks = []
+        for task_data in data:
+            assert len(task_data) == len(headers)
+            supported_benchmarks.append(dict(zip(headers, task_data)))
+
+        if self.json:
+            print(json.dumps({"tasks": supported_benchmarks}, indent=2))
+        else:
+            self._print_table(supported_benchmarks)
+
+    def _print_table(self, tasks: list[dict]) -> None:
+        """Print tasks grouped by harness and container in table format."""
+        if not tasks:
+            print("No tasks found.")
+            return
+
+        # Group tasks by harness and container
+        grouped = defaultdict(lambda: defaultdict(list))
+        for task in tasks:
+            harness = task["harness"]
+            container = task["container"]
+            grouped[harness][container].append(task)
+
+        # Print grouped tables
+        for i, (harness, containers) in enumerate(grouped.items()):
+            if i > 0:
+                print()  # Extra spacing between harnesses
+
+            for j, (container, container_tasks) in enumerate(containers.items()):
+                if j > 0:
+                    print()  # Spacing between containers
+
+                # Prepare task table first to get column widths
+                task_headers = ["task", "endpoint_type"]
+                rows = []
+                for task in container_tasks:
+                    rows.append([task["task"], task["endpoint_type"]])
+
+                # Sort tasks alphabetically for better readability
+                rows.sort(key=lambda x: x[0])
+
+                # Calculate column widths with some padding
+                widths = [
+                    max(len(task_headers[i]), max(len(str(row[i])) for row in rows)) + 2
+                    for i in range(len(task_headers))
+                ]
+
+                # Calculate minimum table width based on task columns
+                min_table_width = sum(widths) + len(widths) + 1
+
+                # Calculate required width for header content
+                harness_line = f"harness: {harness}"
+                container_line = f"container: {container}"
+                header_content_width = (
+                    max(len(harness_line), len(container_line)) + 4
+                )  # +4 for "| " and " |"
+
+                # Use the larger of the two widths
+                table_width = max(min_table_width, header_content_width)
+
+                # Print combined header with harness and container info
+                print("=" * table_width)
+                print(f"{harness_line}")
+                print(f"{container_line}")
+
+                # Adjust column widths to fill the full table width
+                available_width = table_width
+                # Give more space to the first column (task names can be long)
+                adjusted_widths = [
+                    max(
+                        widths[0], available_width * 2 // 3
+                    ),  # 2/3 of available width for task
+                    0,  # Will be calculated as remainder
+                ]
+                adjusted_widths[1] = (
+                    available_width - adjusted_widths[0]
+                )  # Remainder for endpoint_type
+
+                # Print task table header separator
+                print(" " * table_width)
+                header_row = f"{task_headers[0]:<{adjusted_widths[0]}}{task_headers[1]:<{adjusted_widths[1]}}"
+                print(header_row)
+                print("-" * table_width)
+
+                # Print task rows
+                for row in rows:
+                    data_row = f"{str(row[0]):<{adjusted_widths[0]}}{str(row[1]):<{adjusted_widths[1]}}"
+                    print(data_row)
+
+                print("-" * table_width)
+                # Show task count
+                task_count = len(rows)
+                print(f"  {task_count} task{'s' if task_count != 1 else ''} available")
+                print("=" * table_width)
+
+                print()

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/cli/main.py

@@ -70,6 +70,9 @@ def create_parser() -> ArgumentParser:
     ls_parser = subparsers.add_parser(
         "ls", help="List resources", description="List tasks or runs"
     )
+    # Add arguments from `ls tasks` so that they work with `ls` as default alias
+    ls_parser.add_arguments(ls_tasks.Cmd, dest="tasks_alias")
+
     ls_sub = ls_parser.add_subparsers(dest="ls_command", required=False)
 
     # ls tasks (default)
@@ -126,12 +129,10 @@ def main() -> None:
         # Dispatch nested ls subcommands
         if args.ls_command is None or args.ls_command == "tasks":
             # Default to tasks when no subcommand specified
-            if hasattr(args, "tasks"):
-                args.tasks.execute()
+            if hasattr(args, "tasks_alias"):
+                args.tasks_alias.execute()
             else:
-                # Create default tasks command if not specified
-                tasks_cmd = ls_tasks.Cmd()
-                tasks_cmd.execute()
+                args.tasks.execute()
         elif args.ls_command == "runs":
             args.runs.execute()
     elif args.command == "export":

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/cli/run.py

@@ -17,12 +17,8 @@ import pathlib
 import time
 from dataclasses import dataclass
 
-import yaml
-from omegaconf import OmegaConf
 from simple_parsing import field
 
-from nemo_evaluator_launcher.api.functional import RunConfig, run_eval
-
 
 @dataclass
 class Cmd:
@@ -65,6 +61,12 @@ class Cmd:
     )
 
     def execute(self) -> None:
+        # Import heavy dependencies only when needed
+        import yaml
+        from omegaconf import OmegaConf
+
+        from nemo_evaluator_launcher.api.functional import RunConfig, run_eval
+
         # Load configuration either from Hydra or from a run config file
         if self.run_config_file:
             # Validate that run config file is not used with other config options

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/cli/status.py

@@ -13,13 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import json
 from dataclasses import dataclass
 
 from simple_parsing import field
 
-from nemo_evaluator_launcher.api.functional import get_status
-
 
 @dataclass
 class Cmd:
@@ -36,6 +33,11 @@ class Cmd:
     )
 
     def execute(self) -> None:
+        # Import heavy dependencies only when needed
+        import json
+
+        from nemo_evaluator_launcher.api.functional import get_status
+
         res = get_status(self.job_ids)
         if self.json:
             # Remove progress field from JSON output as it's a WIP feature

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/executors/lepton/job_helpers.py

@@ -23,13 +23,6 @@ import subprocess
 import time
 from typing import Any, List, Union
 
-from leptonai.api.v1.types.affinity import LeptonResourceAffinity
-from leptonai.api.v1.types.common import LeptonVisibility, Metadata
-from leptonai.api.v1.types.deployment import EnvVar, LeptonContainer, Mount
-from leptonai.api.v1.types.job import LeptonJob, LeptonJobUserSpec
-
-# Import lepton dependencies
-from leptonai.api.v2.client import APIClient
 from omegaconf import DictConfig
 
 from nemo_evaluator_launcher.common.logging_utils import logger
@@ -92,6 +85,18 @@ def _create_lepton_job_api(
 ) -> tuple[bool, str]:
     """Create Lepton job using API client (preferred method)."""
     try:
+        # Import leptonai dependencies locally
+        from leptonai.api.v1.types.affinity import LeptonResourceAffinity
+        from leptonai.api.v1.types.common import LeptonVisibility, Metadata
+        from leptonai.api.v1.types.deployment import (
+            EnvValue,
+            EnvVar,
+            LeptonContainer,
+            Mount,
+        )
+        from leptonai.api.v1.types.job import LeptonJob, LeptonJobUserSpec
+        from leptonai.api.v2.client import APIClient
+
         client = APIClient()
 
         # Prepare environment variables (support both direct values and secret references)
@@ -99,12 +104,8 @@ def _create_lepton_job_api(
         if env_vars:
             for key, value in env_vars.items():
                 # Handle both regular dicts and OmegaConf objects
-                from omegaconf import DictConfig
-
                 if isinstance(value, (dict, DictConfig)) and "value_from" in value:
                     # Secret reference: {value_from: {secret_name_ref: "secret_name"}}
-                    from leptonai.api.v1.types.deployment import EnvValue
-
                     # Convert OmegaConf to dict if needed
                     value_dict = dict(value) if isinstance(value, DictConfig) else value
                     env_var = EnvVar(
@@ -203,6 +204,9 @@ def get_lepton_job_status(job_name_or_id: str) -> dict[Any, Any] | None:
 def _get_lepton_job_status_api(job_name_or_id: str) -> dict[Any, Any] | None:
     """Get job status using API client (preferred method)."""
     try:
+        # Import leptonai dependencies locally
+        from leptonai.api.v2.client import APIClient
+
         client = APIClient()
 
         # Try to get job by ID first, then by name

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher/package_info.py

@@ -16,8 +16,8 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 0
-PRE_RELEASE = "rc9"
+PATCH = 2
+PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)
 VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE)

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc9
+Version: 0.1.2
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.2}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt

@@ -15,7 +15,6 @@ src/nemo_evaluator_launcher/api/types.py
 src/nemo_evaluator_launcher/api/utils.py
 src/nemo_evaluator_launcher/cli/__init__.py
 src/nemo_evaluator_launcher/cli/export.py
-src/nemo_evaluator_launcher/cli/info.py
 src/nemo_evaluator_launcher/cli/kill.py
 src/nemo_evaluator_launcher/cli/ls_runs.py
 src/nemo_evaluator_launcher/cli/ls_tasks.py

nemo_evaluator_launcher-0.1.0rc9/src/nemo_evaluator_launcher/cli/info.py

@@ -1,117 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import datetime as _dt
-import sys
-from dataclasses import dataclass
-from typing import Optional
-
-from simple_parsing import field
-
-from nemo_evaluator_launcher.api.functional import (
-    get_invocation_benchmarks,
-    list_all_invocations_summary,
-)
-
-
-@dataclass
-class Cmd:
-    """Show a concise invocations table from the exec DB."""
-
-    limit: Optional[int] = field(default=None, alias=["--limit"], help="Max rows")
-    executor: Optional[str] = field(
-        default=None, alias=["--executor"], help="Filter by executor"
-    )
-    since: Optional[str] = field(
-        default=None,
-        alias=["--since"],
-        help="Filter by ISO date/time (e.g., 2025-08-20 or 2025-08-20T12:00:00)",
-    )
-
-    def execute(self) -> None:
-        rows = list_all_invocations_summary()
-
-        # Apply filters
-        if self.executor:
-            rows = [
-                r
-                for r in rows
-                if (r.get("executor") or "").lower() == self.executor.lower()
-            ]
-
-        if self.since:
-            try:
-                # Parse with optional time
-                if "T" in self.since:
-                    since_ts = _dt.datetime.fromisoformat(self.since).timestamp()
-                else:
-                    since_ts = _dt.datetime.fromisoformat(
-                        self.since + "T00:00:00"
-                    ).timestamp()
-                rows = [r for r in rows if (r.get("earliest_job_ts") or 0) >= since_ts]
-            except Exception:
-                print(
-                    f"Invalid --since value: {self.since}. Use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS",
-                    file=sys.stderr,
-                )
-                sys.exit(2)
-
-        if self.limit is not None and self.limit >= 0:
-            rows = rows[: self.limit]
-
-        # Print table (always include benchmarks column)
-        header = [
-            "invocation_id",
-            "earliest_job_ts",
-            "num_jobs",
-            "executor",
-            "benchmarks",
-        ]
-        # Format ts as ISO without microseconds
-        table_rows = []
-        for r in rows:
-            ts = r.get("earliest_job_ts", 0) or 0
-            try:
-                ts_iso = (
-                    _dt.datetime.fromtimestamp(ts).replace(microsecond=0).isoformat()
-                )
-            except Exception:
-                ts_iso = ""
-            inv = r.get("invocation_id", "")
-            try:
-                bmarks = get_invocation_benchmarks(inv)
-                bmarks_cell = ",".join(bmarks) if bmarks else "unknown"
-            except Exception:
-                bmarks_cell = "unknown"
-            table_rows.append(
-                [
-                    str(inv),
-                    ts_iso,
-                    str(r.get("num_jobs", 0)),
-                    str(r.get("executor", "")),
-                    bmarks_cell,
-                ]
-            )
-
-        widths = [len(h) for h in header]
-        for tr in table_rows:
-            for i, cell in enumerate(tr):
-                if len(cell) > widths[i]:
-                    widths[i] = len(cell)
-        fmt = "  ".join([f"{{:<{w}}}" for w in widths])
-        print(fmt.format(*header))
-        print("  ".join(["-" * w for w in widths]))
-        for tr in table_rows:
-            print(fmt.format(*tr))

nemo_evaluator_launcher-0.1.0rc9/src/nemo_evaluator_launcher/cli/ls_tasks.py

@@ -1,34 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import json
-from dataclasses import dataclass
-
-from nemo_evaluator_launcher.api.functional import get_tasks_list
-
-
-@dataclass
-class Cmd:
-    """List command configuration."""
-
-    def execute(self) -> None:
-        # TODO(dfridman): modify `get_tasks_list` to return a list of dicts in the first place
-        data = get_tasks_list()
-        headers = ["task", "endpoint_type", "harness", "container"]
-        supported_benchmarks = []
-        for task_data in data:
-            assert len(task_data) == len(headers)
-            supported_benchmarks.append(dict(zip(headers, task_data)))
-        print(json.dumps(supported_benchmarks, indent=2))