nemo-evaluator-launcher 0.1.44__tar.gz → 0.1.64__tar.gz

{nemo_evaluator_launcher-0.1.44 → nemo_evaluator_launcher-0.1.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.44
+Version: 0.1.64
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -467,6 +467,7 @@ License-File: LICENSE
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: jinja2<4.0.0,>=3.1.6
 Requires-Dist: leptonai>=0.25.0
+Requires-Dist: nemo-evaluator
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: requests>=2.32.4
 Requires-Dist: simple-parsing<0.2.0,>=0.1.7

{nemo_evaluator_launcher-0.1.44 → nemo_evaluator_launcher-0.1.64}/pyproject.toml

@@ -4,6 +4,7 @@ dependencies = [
   "hydra-core>=1.3.2,<2.0.0",
   "jinja2>=3.1.6,<4.0.0",
   "leptonai>=0.25.0",
+  "nemo-evaluator",
   "pyyaml>=6.0.0",
   "requests>=2.32.4",
   "simple-parsing>=0.1.7,<0.2.0",

{nemo_evaluator_launcher-0.1.44 → nemo_evaluator_launcher-0.1.64}/src/nemo_evaluator_launcher/api/functional.py

@@ -18,6 +18,7 @@
 This module provides the main functional entry points for running evaluations, querying job status, and listing available tasks. These functions are intended to be used by CLI commands and external integrations.
 """
 
+import copy
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
@@ -35,7 +36,7 @@ def get_tasks_list() -> list[list[Any]]:
     """Get a list of available tasks from the mapping.
 
     Returns:
-        list[list[Any]]: Each sublist contains task name, endpoint type, harness, and container.
+        list[list[Any]]: Each sublist contains task name, endpoint type, harness, container, arch, description, and type.
     """
     mapping = load_tasks_mapping()
     data = [
@@ -44,6 +45,9 @@ def get_tasks_list() -> list[list[Any]]:
             task_data.get("endpoint_type"),
             task_data.get("harness"),
             task_data.get("container"),
+            task_data.get("arch", ""),
+            task_data.get("description", ""),
+            task_data.get("type", ""),
         ]
         for task_data in mapping.values()
     ]
@@ -75,12 +79,54 @@ def _validate_no_missing_values(cfg: Any, path: str = "") -> None:
             _validate_no_missing_values(value, current_path)
 
 
-def run_eval(cfg: RunConfig, dry_run: bool = False) -> Optional[str]:
+def filter_tasks(cfg: RunConfig, task_names: list[str]) -> RunConfig:
+    """Filter evaluation tasks to only include specified task names.
+
+    Args:
+        cfg: The configuration object for the evaluation run.
+        task_names: List of task names to include (e.g., ["ifeval", "gsm8k"]).
+
+    Returns:
+        RunConfig: A new configuration with filtered tasks (input is not mutated).
+
+    Raises:
+        ValueError: If any requested task is not found in config or no tasks defined.
+    """
+    if not task_names:
+        return cfg
+
+    if not hasattr(cfg.evaluation, "tasks") or not cfg.evaluation.tasks:
+        raise ValueError("No tasks defined in config. Cannot filter tasks.")
+
+    requested_tasks = set(task_names)
+    original_tasks = cfg.evaluation.tasks
+    filtered_tasks = [task for task in original_tasks if task.name in requested_tasks]
+
+    # Fail if ANY requested tasks are not found
+    found_names = {task.name for task in filtered_tasks}
+    not_found = requested_tasks - found_names
+    if not_found:
+        available = [task.name for task in original_tasks]
+        raise ValueError(
+            f"Requested task(s) not found in config: {sorted(not_found)}. "
+            f"Available tasks: {available}"
+        )
+
+    # Create a deep copy to preserve input immutability
+    result = copy.deepcopy(cfg)
+    result.evaluation.tasks = filtered_tasks
+    return result
+
+
+def run_eval(
+    cfg: RunConfig, dry_run: bool = False, tasks: Optional[list[str]] = None
+) -> Optional[str]:
     """Run evaluation with specified config and overrides.
 
     Args:
         cfg: The configuration object for the evaluation run.
         dry_run: If True, do not run the evaluation, just prepare scripts and save them.
+        tasks: Optional list of task names to run. If provided, only these tasks will be executed.
 
     Returns:
         Optional[str]: The invocation ID for the evaluation run.
@@ -89,6 +135,10 @@ def run_eval(cfg: RunConfig, dry_run: bool = False) -> Optional[str]:
         ValueError: If configuration validation fails or MISSING values are found.
         RuntimeError: If the executor fails to start the evaluation.
     """
+    # Filter tasks if specified
+    if tasks:
+        cfg = filter_tasks(cfg, tasks)
+
     # Validate that no MISSING values exist in the configuration
     _validate_no_missing_values(cfg)
 
@@ -285,7 +335,7 @@ def stream_logs(
 
     # Collect all jobs from all IDs, grouped by executor
     executor_to_jobs: Dict[str, Dict[str, JobData]] = {}
-    executor_to_invocations: Dict[str, List[str]] = {}
+    executor_to_invocations: Dict[str, list[str]] = {}
 
     # TODO(agronskiy): refactor the `.`-checking job in all the functions.
     for id_or_prefix in ids_or_prefixes:
@@ -517,7 +567,7 @@ def kill_job_or_invocation(id: str) -> list[dict[str, Any]]:
 
 
 def export_results(
-    invocation_ids: Union[str, List[str]],
+    invocation_ids: Union[str, list[str]],
     dest: str = "local",
     config: dict[Any, Any] | None = None,
 ) -> dict:

{nemo_evaluator_launcher-0.1.44 → nemo_evaluator_launcher-0.1.64}/src/nemo_evaluator_launcher/api/types.py

@@ -18,7 +18,7 @@
 This module defines data structures and helpers for configuration and type safety in the API layer.
 """
 
-import os
+import pathlib
 import warnings
 from dataclasses import dataclass
 from typing import cast
@@ -42,33 +42,40 @@ from nemo_evaluator_launcher.common.logging_utils import logger
 class RunConfig(DictConfig):
     @staticmethod
     def from_hydra(
-        config_name: str = "default",
-        config_dir: str | None = None,
-        hydra_overrides: list[str] = [],
-        dict_overrides: dict = {},
+        config: str | None = None,
+        hydra_overrides: list[str] | None = None,
+        dict_overrides: dict | None = None,
     ) -> "RunConfig":
         """Load configuration from Hydra and merge with dictionary overrides.
 
         Args:
-            config_name: Name of the Hydra configuration to load.
+            config: Optional full path to a config file (e.g. /path/to/my_config.yaml).
+                    If omitted, loads the internal default config from
+                    `nemo_evaluator_launcher.configs`.
             hydra_overrides: List of Hydra command-line style overrides.
             dict_overrides: Dictionary of configuration overrides to merge.
-            config_dir: Optional path to user config directory. If provided, Hydra will
-                       search in this directory first, then fall back to internal configs.
 
         Returns:
             RunConfig: Merged configuration object.
         """
-        overrides = hydra_overrides.copy()
+        overrides = list(hydra_overrides or [])
+        dict_overrides = dict_overrides or {}
+
+        resolved_config_path: str | None = None
+        config_name = "default"
+
         # Check if a GlobalHydra instance is already initialized and clear it
         if GlobalHydra.instance().is_initialized():
             GlobalHydra.instance().clear()
 
-        if config_dir:
-            # Convert relative path to absolute path if needed
-            if not os.path.isabs(config_dir):
-                config_dir = os.path.abspath(config_dir)
+        if config:
+            config_path = pathlib.Path(config).expanduser()
+            if not config_path.is_absolute():
+                config_path = (pathlib.Path.cwd() / config_path).resolve()
+            resolved_config_path = str(config_path)
 
+            config_dir = str(config_path.parent)
+            config_name = str(config_path.stem)
             hydra.initialize_config_dir(
                 config_dir=config_dir,
                 version_base=None,
@@ -90,7 +97,7 @@ class RunConfig(DictConfig):
         logger.debug(
             "Loaded run config from hydra",
             config_name=config_name,
-            config_dir=config_dir,
+            config=resolved_config_path,
             overrides=hydra_overrides,
             dict_overrides=dict_overrides,
             result=cfg,

nemo_evaluator_launcher-0.1.64/src/nemo_evaluator_launcher/cli/ls_task.py

@@ -0,0 +1,280 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""CLI command for listing task details."""
+
+import json
+from dataclasses import dataclass
+
+import yaml
+from simple_parsing import field
+
+from nemo_evaluator_launcher.common.container_metadata import (
+    TaskIntermediateRepresentation,
+    load_tasks_from_tasks_file,
+)
+from nemo_evaluator_launcher.common.logging_utils import logger
+from nemo_evaluator_launcher.common.mapping import load_tasks_mapping
+from nemo_evaluator_launcher.common.printing_utils import (
+    bold,
+    cyan,
+    magenta,
+    yellow,
+)
+
+
+@dataclass
+class Cmd:
+    """List task command configuration."""
+
+    task_identifier: str = field(
+        default="",
+        positional=True,
+        help="Task identifier in format '[harness.]task_name'. If empty, shows all tasks.",
+    )
+    json: bool = field(
+        default=False,
+        action="store_true",
+        help="Print output as JSON instead of formatted text",
+    )
+    tasks_file: str = field(
+        default="",
+        help="Path to all_tasks_irs.yaml file (default: auto-detect)",
+    )
+    from_container: str = field(
+        default="",
+        help="Load tasks from container image (e.g., nvcr.io/nvidia/eval-factory/simple-evals:25.10). "
+        "If provided, extracts framework.yml from container and loads tasks on-the-fly instead of using all_tasks_irs.yaml",
+    )
+
+    def execute(self) -> None:
+        """Execute the ls task command."""
+        import pathlib
+
+        # Initialize tasks_path to None - it will be set when loading from file
+        tasks_path = None
+
+        # If --from is provided, load tasks from container
+        if self.from_container:
+            from nemo_evaluator_launcher.common.container_metadata import (
+                load_tasks_from_container,
+            )
+
+            try:
+                tasks = load_tasks_from_container(self.from_container)
+            except ValueError as e:
+                print(f"Error: {e}")
+                return
+            except Exception as e:
+                logger.error(
+                    "Failed to load tasks from container",
+                    container=self.from_container,
+                    error=str(e),
+                    exc_info=True,
+                )
+                return
+
+            if not tasks:
+                logger.error(
+                    "No tasks found in container",
+                    container=self.from_container,
+                )
+                return
+
+            logger.debug(
+                "Loaded tasks from container",
+                container=self.from_container,
+                num_tasks=len(tasks),
+                containers=set(task.container for task in tasks),
+            )
+            mapping_verified = True  # Tasks from container are always verified
+        else:
+            # Default behavior: load from all_tasks_irs.yaml
+            if self.tasks_file:
+                tasks_path = pathlib.Path(self.tasks_file)
+                if not tasks_path.exists():
+                    logger.error("Tasks file not found", path=str(tasks_path))
+                    return
+
+            # Load tasks
+            try:
+                tasks, mapping_verified = load_tasks_from_tasks_file(tasks_path)
+            except Exception as e:
+                print(f"Error loading tasks: {e}")
+                import traceback
+
+                traceback.print_exc()
+                logger.error("Failed to load tasks", error=str(e), exc_info=True)
+                return
+
+            # Display warning if mapping is not verified
+            if not mapping_verified:
+                print(
+                    yellow(
+                        "⚠ Warning: Tasks are from unverified mapping (mapping.toml checksum mismatch)"
+                    )
+                )
+                print(
+                    yellow(
+                        "  Consider regenerating all_tasks_irs.yaml if mapping.toml has changed"
+                    )
+                )
+                print()
+
+            # Override containers from mapping.toml (which has the latest containers)
+            # This ensures ls task shows the same containers as ls tasks
+            # Only do this when NOT using --from (when loading from all_tasks_irs.yaml)
+            try:
+                mapping = load_tasks_mapping()
+                # Create a lookup: (normalized_harness, normalized_task_name) -> container
+                # Use case-insensitive keys for matching
+                container_lookup = {}
+                for (harness, task_name), task_data in mapping.items():
+                    container = task_data.get("container")
+                    if container:
+                        # Normalize harness name for lookup (frameworks.yaml uses hyphens)
+                        normalized_harness = harness.replace("_", "-").lower()
+                        normalized_task = task_name.lower()
+                        container_lookup[(normalized_harness, normalized_task)] = (
+                            container
+                        )
+
+                # Update task containers from mapping.toml
+                for task in tasks:
+                    # Defensive checks: ensure task has required attributes
+                    if not hasattr(task, "harness") or not task.harness:
+                        logger.warning(
+                            "Task missing harness attribute, skipping container override",
+                            task_name=getattr(task, "name", "unknown"),
+                        )
+                        continue
+                    if not hasattr(task, "name") or not task.name:
+                        logger.warning(
+                            "Task missing name attribute, skipping container override",
+                            harness=getattr(task, "harness", "unknown"),
+                        )
+                        continue
+
+                    # Normalize both harness and task name for case-insensitive lookup
+                    normalized_harness = task.harness.lower()
+                    normalized_task = task.name.lower()
+                    lookup_key = (normalized_harness, normalized_task)
+                    if lookup_key in container_lookup:
+                        task.container = container_lookup[lookup_key]
+            except Exception as e:
+                logger.debug(
+                    "Failed to override containers from mapping.toml",
+                    error=str(e),
+                )
+                # Continue with containers from all_tasks_irs.yaml if mapping load fails
+
+        if not tasks:
+            print("No tasks found.")
+            if tasks_path:
+                print(f"  Tasks file: {tasks_path}")
+            else:
+                print(
+                    "  Note: Make sure all_tasks_irs.yaml exists and contains valid task definitions."
+                )
+            return
+
+        # Parse task identifier
+        harness_filter = None
+        task_filter = None
+        if self.task_identifier:
+            if "." in self.task_identifier:
+                parts = self.task_identifier.split(".", 1)
+                harness_filter = parts[0]
+                task_filter = parts[1]
+            else:
+                task_filter = self.task_identifier
+
+        # Filter tasks
+        filtered_tasks = []
+        for task in tasks:
+            if harness_filter and task.harness.lower() != harness_filter.lower():
+                continue
+            if task_filter and task.name.lower() != task_filter.lower():
+                continue
+            filtered_tasks.append(task)
+
+        if not filtered_tasks:
+            print(f"No tasks found matching: {self.task_identifier}")
+            if self.task_identifier:
+                # Show available tasks for debugging
+                print("\nAvailable tasks (showing first 10):")
+                for i, task in enumerate(tasks[:10]):
+                    print(f"  - {task.harness}.{task.name}")
+                if len(tasks) > 10:
+                    print(f"  ... and {len(tasks) - 10} more")
+            return
+
+        # Display tasks
+        if self.json:
+            self._print_json(filtered_tasks)
+        else:
+            self._print_formatted(filtered_tasks, mapping_verified)
+
+    def _print_json(self, tasks: list[TaskIntermediateRepresentation]) -> None:
+        """Print tasks as JSON."""
+        tasks_dict = [task.to_dict() for task in tasks]
+        print(json.dumps({"tasks": tasks_dict}, indent=2))
+
+    def _print_formatted(
+        self, tasks: list[TaskIntermediateRepresentation], mapping_verified: bool = True
+    ) -> None:
+        """Print tasks in formatted text with colorized output."""
+        for i, task in enumerate(tasks):
+            if i > 0:
+                print()  # Spacing between tasks
+                print(bold("=" * 80))
+
+            # Task name - bold and magenta key, cyan value (matching logging utils)
+            print(f"{bold(magenta('Task:'))} {bold(cyan(str(task.name)))}")
+
+            # Description - magenta key, cyan value
+            if task.description:
+                print(f"{magenta('Description:')} {cyan(str(task.description))}")
+
+            # Harness - magenta key, cyan value
+            print(f"{magenta('Harness:')} {cyan(str(task.harness))}")
+
+            # Container - magenta key, cyan value
+            print(f"{magenta('Container:')} {cyan(str(task.container))}")
+
+            # Container Digest - magenta key, cyan value
+            if task.container_digest:
+                print(
+                    f"{magenta('Container Digest:')} {cyan(str(task.container_digest))}"
+                )
+
+            # Print defaults as YAML
+            if task.defaults:
+                print(f"\n{bold(magenta('Defaults:'))}")
+                defaults_yaml = yaml.dump(
+                    task.defaults, default_flow_style=False, sort_keys=False
+                )
+                # Indent defaults - use cyan for YAML content (FDF values)
+                for line in defaults_yaml.split("\n"):
+                    if line.strip():
+                        print(f"  {cyan(line)}")
+                    else:
+                        print()
+
+            print(bold("-" * 80))
+
+        # Total count - bold
+        task_word = "task" if len(tasks) == 1 else "tasks"
+        print(f"\n{bold(f'Total: {len(tasks)} {task_word}')}")