nemo-evaluator-launcher 0.1.0rc9__tar.gz → 0.1.1__tar.gz

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc9
+Version: 0.1.1
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

nemo_evaluator_launcher-0.1.1/src/nemo_evaluator_launcher/cli/ls_tasks.py

@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+from collections import defaultdict
+from dataclasses import dataclass
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import get_tasks_list
+
+
+@dataclass
+class Cmd:
+    """List command configuration."""
+
+    json: bool = field(
+        default=False,
+        action="store_true",
+        help="Print output as JSON instead of table format",
+    )
+
+    def execute(self) -> None:
+        # TODO(dfridman): modify `get_tasks_list` to return a list of dicts in the first place
+        data = get_tasks_list()
+        headers = ["task", "endpoint_type", "harness", "container"]
+        supported_benchmarks = []
+        for task_data in data:
+            assert len(task_data) == len(headers)
+            supported_benchmarks.append(dict(zip(headers, task_data)))
+
+        if self.json:
+            print(json.dumps({"tasks": supported_benchmarks}, indent=2))
+        else:
+            self._print_table(supported_benchmarks)
+
+    def _print_table(self, tasks: list[dict]) -> None:
+        """Print tasks grouped by harness and container in table format."""
+        if not tasks:
+            print("No tasks found.")
+            return
+
+        # Group tasks by harness and container
+        grouped = defaultdict(lambda: defaultdict(list))
+        for task in tasks:
+            harness = task["harness"]
+            container = task["container"]
+            grouped[harness][container].append(task)
+
+        # Print grouped tables
+        for i, (harness, containers) in enumerate(grouped.items()):
+            if i > 0:
+                print()  # Extra spacing between harnesses
+
+            for j, (container, container_tasks) in enumerate(containers.items()):
+                if j > 0:
+                    print()  # Spacing between containers
+
+                # Prepare task table first to get column widths
+                task_headers = ["task", "endpoint_type"]
+                rows = []
+                for task in container_tasks:
+                    rows.append([task["task"], task["endpoint_type"]])
+
+                # Sort tasks alphabetically for better readability
+                rows.sort(key=lambda x: x[0])
+
+                # Calculate column widths with some padding
+                widths = [
+                    max(len(task_headers[i]), max(len(str(row[i])) for row in rows)) + 2
+                    for i in range(len(task_headers))
+                ]
+
+                # Calculate minimum table width based on task columns
+                min_table_width = sum(widths) + len(widths) + 1
+
+                # Calculate required width for header content
+                harness_line = f"harness: {harness}"
+                container_line = f"container: {container}"
+                header_content_width = (
+                    max(len(harness_line), len(container_line)) + 4
+                )  # +4 for "| " and " |"
+
+                # Use the larger of the two widths
+                table_width = max(min_table_width, header_content_width)
+
+                # Print combined header with harness and container info
+                print("=" * table_width)
+                print(f"{harness_line}")
+                print(f"{container_line}")
+
+                # Adjust column widths to fill the full table width
+                available_width = table_width
+                # Give more space to the first column (task names can be long)
+                adjusted_widths = [
+                    max(
+                        widths[0], available_width * 2 // 3
+                    ),  # 2/3 of available width for task
+                    0,  # Will be calculated as remainder
+                ]
+                adjusted_widths[1] = (
+                    available_width - adjusted_widths[0]
+                )  # Remainder for endpoint_type
+
+                # Print task table header separator
+                print(" " * table_width)
+                header_row = f"{task_headers[0]:<{adjusted_widths[0]}}{task_headers[1]:<{adjusted_widths[1]}}"
+                print(header_row)
+                print("-" * table_width)
+
+                # Print task rows
+                for row in rows:
+                    data_row = f"{str(row[0]):<{adjusted_widths[0]}}{str(row[1]):<{adjusted_widths[1]}}"
+                    print(data_row)
+
+                print("-" * table_width)
+                # Show task count
+                task_count = len(rows)
+                print(f"  {task_count} task{'s' if task_count != 1 else ''} available")
+                print("=" * table_width)
+
+                print()

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.1}/src/nemo_evaluator_launcher/cli/main.py

@@ -70,6 +70,9 @@ def create_parser() -> ArgumentParser:
     ls_parser = subparsers.add_parser(
         "ls", help="List resources", description="List tasks or runs"
     )
+    # Add arguments from `ls tasks` so that they work with `ls` as default alias
+    ls_parser.add_arguments(ls_tasks.Cmd, dest="tasks_alias")
+
     ls_sub = ls_parser.add_subparsers(dest="ls_command", required=False)
 
     # ls tasks (default)
@@ -126,12 +129,10 @@ def main() -> None:
         # Dispatch nested ls subcommands
         if args.ls_command is None or args.ls_command == "tasks":
             # Default to tasks when no subcommand specified
-            if hasattr(args, "tasks"):
-                args.tasks.execute()
+            if hasattr(args, "tasks_alias"):
+                args.tasks_alias.execute()
             else:
-                # Create default tasks command if not specified
-                tasks_cmd = ls_tasks.Cmd()
-                tasks_cmd.execute()
+                args.tasks.execute()
         elif args.ls_command == "runs":
             args.runs.execute()
     elif args.command == "export":

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.1}/src/nemo_evaluator_launcher/package_info.py

@@ -16,8 +16,8 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 0
-PRE_RELEASE = "rc9"
+PATCH = 1
+PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)
 VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE)

{nemo_evaluator_launcher-0.1.0rc9 → nemo_evaluator_launcher-0.1.1}/src/nemo_evaluator_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc9
+Version: 0.1.1
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

nemo_evaluator_launcher-0.1.0rc9/src/nemo_evaluator_launcher/cli/ls_tasks.py

@@ -1,34 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import json
-from dataclasses import dataclass
-
-from nemo_evaluator_launcher.api.functional import get_tasks_list
-
-
-@dataclass
-class Cmd:
-    """List command configuration."""
-
-    def execute(self) -> None:
-        # TODO(dfridman): modify `get_tasks_list` to return a list of dicts in the first place
-        data = get_tasks_list()
-        headers = ["task", "endpoint_type", "harness", "container"]
-        supported_benchmarks = []
-        for task_data in data:
-            assert len(task_data) == len(headers)
-            supported_benchmarks.append(dict(zip(headers, task_data)))
-        print(json.dumps(supported_benchmarks, indent=2))