nemo-evaluator-launcher 0.1.21__tar.gz → 0.1.64__tar.gz

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.21
+Version: 0.1.64
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -458,7 +458,7 @@ License:
               OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
               SOFTWARE.
         
-Project-URL: homepage, https://github.com/NVIDIA-NeMo/Eval
+Project-URL: homepage, https://github.com/NVIDIA-NeMo/Evaluator
 Project-URL: repository, https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-launcher
 Keywords: deep learning,evaluations,machine learning,gpu,NLP,pytorch,torch
 Requires-Python: <3.14,>=3.10
@@ -467,6 +467,7 @@ License-File: LICENSE
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: jinja2<4.0.0,>=3.1.6
 Requires-Dist: leptonai>=0.25.0
+Requires-Dist: nemo-evaluator
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: requests>=2.32.4
 Requires-Dist: simple-parsing<0.2.0,>=0.1.7
@@ -478,7 +479,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
 Provides-Extra: wandb
 Requires-Dist: wandb>=0.15.0; extra == "wandb"
 Provides-Extra: gsheets
-Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
+Requires-Dist: gspread>=5.0.0; extra == "gsheets"
 Provides-Extra: exporters
 Requires-Dist: mlflow; extra == "exporters"
 Requires-Dist: wandb; extra == "exporters"

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.64}/pyproject.toml

@@ -4,6 +4,7 @@ dependencies = [
   "hydra-core>=1.3.2,<2.0.0",
   "jinja2>=3.1.6,<4.0.0",
   "leptonai>=0.25.0",
+  "nemo-evaluator",
   "pyyaml>=6.0.0",
   "requests>=2.32.4",
   "simple-parsing>=0.1.7,<0.2.0",
@@ -33,20 +34,21 @@ keywords = [
 
 [project.urls]
 # BEGIN(if-changed): check package_info.py
-homepage = "https://github.com/NVIDIA-NeMo/Eval"
+homepage = "https://github.com/NVIDIA-NeMo/Evaluator"
 repository = "https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-launcher"
 # END(if-changed)
 
 [project.optional-dependencies]
 mlflow = ["mlflow>=2.8.0"]
 wandb = ["wandb>=0.15.0"]
-gsheets = ["gsheets>=0.1.0"]
+gsheets = ["gspread>=5.0.0"]
 exporters = ["mlflow", "wandb", "gsheets"]
 all = ["mlflow", "wandb", "gsheets"]
 
 [project.scripts]
 nemo-evaluator-launcher = "nemo_evaluator_launcher.cli.main:main"
 nv-eval = "nemo_evaluator_launcher.cli.main:main"
+nel = "nemo_evaluator_launcher.cli.main:main"
 
 [dependency-groups]
 dev = [
@@ -75,6 +77,7 @@ where = ["src"]
   "resources/**/*",
   "configs/**/*",
   "executors/**/*.sh",
+  "executors/**/*.template",
 ]
 
 [tool.setuptools.dynamic]

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.64}/src/nemo_evaluator_launcher/api/functional.py

@@ -18,8 +18,9 @@
 This module provides the main functional entry points for running evaluations, querying job status, and listing available tasks. These functions are intended to be used by CLI commands and external integrations.
 """
 
+import copy
 from pathlib import Path
-from typing import Any, List, Optional, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import yaml
 from omegaconf import DictConfig, OmegaConf
@@ -35,7 +36,7 @@ def get_tasks_list() -> list[list[Any]]:
     """Get a list of available tasks from the mapping.
 
     Returns:
-        list[list[Any]]: Each sublist contains task name, endpoint type, harness, and container.
+        list[list[Any]]: Each sublist contains task name, endpoint type, harness, container, arch, description, and type.
     """
     mapping = load_tasks_mapping()
     data = [
@@ -44,6 +45,9 @@ def get_tasks_list() -> list[list[Any]]:
             task_data.get("endpoint_type"),
             task_data.get("harness"),
             task_data.get("container"),
+            task_data.get("arch", ""),
+            task_data.get("description", ""),
+            task_data.get("type", ""),
         ]
         for task_data in mapping.values()
     ]
@@ -75,12 +79,54 @@ def _validate_no_missing_values(cfg: Any, path: str = "") -> None:
             _validate_no_missing_values(value, current_path)
 
 
-def run_eval(cfg: RunConfig, dry_run: bool = False) -> Optional[str]:
+def filter_tasks(cfg: RunConfig, task_names: list[str]) -> RunConfig:
+    """Filter evaluation tasks to only include specified task names.
+
+    Args:
+        cfg: The configuration object for the evaluation run.
+        task_names: List of task names to include (e.g., ["ifeval", "gsm8k"]).
+
+    Returns:
+        RunConfig: A new configuration with filtered tasks (input is not mutated).
+
+    Raises:
+        ValueError: If any requested task is not found in config or no tasks defined.
+    """
+    if not task_names:
+        return cfg
+
+    if not hasattr(cfg.evaluation, "tasks") or not cfg.evaluation.tasks:
+        raise ValueError("No tasks defined in config. Cannot filter tasks.")
+
+    requested_tasks = set(task_names)
+    original_tasks = cfg.evaluation.tasks
+    filtered_tasks = [task for task in original_tasks if task.name in requested_tasks]
+
+    # Fail if ANY requested tasks are not found
+    found_names = {task.name for task in filtered_tasks}
+    not_found = requested_tasks - found_names
+    if not_found:
+        available = [task.name for task in original_tasks]
+        raise ValueError(
+            f"Requested task(s) not found in config: {sorted(not_found)}. "
+            f"Available tasks: {available}"
+        )
+
+    # Create a deep copy to preserve input immutability
+    result = copy.deepcopy(cfg)
+    result.evaluation.tasks = filtered_tasks
+    return result
+
+
+def run_eval(
+    cfg: RunConfig, dry_run: bool = False, tasks: Optional[list[str]] = None
+) -> Optional[str]:
     """Run evaluation with specified config and overrides.
 
     Args:
         cfg: The configuration object for the evaluation run.
         dry_run: If True, do not run the evaluation, just prepare scripts and save them.
+        tasks: Optional list of task names to run. If provided, only these tasks will be executed.
 
     Returns:
         Optional[str]: The invocation ID for the evaluation run.
@@ -89,6 +135,10 @@ def run_eval(cfg: RunConfig, dry_run: bool = False) -> Optional[str]:
         ValueError: If configuration validation fails or MISSING values are found.
         RuntimeError: If the executor fails to start the evaluation.
     """
+    # Filter tasks if specified
+    if tasks:
+        cfg = filter_tasks(cfg, tasks)
+
     # Validate that no MISSING values exist in the configuration
     _validate_no_missing_values(cfg)
 
@@ -116,6 +166,7 @@ def get_status(ids_or_prefixes: list[str]) -> list[dict[str, Any]]:
     db = ExecutionDB()
     results: List[dict[str, Any]] = []
 
+    # TODO(agronskiy): refactor the `.`-checking job in all the functions.
     for id_or_prefix in ids_or_prefixes:
         # If id looks like an invocation_id (no dot), get all jobs for it
         if "." not in id_or_prefix:
@@ -259,6 +310,108 @@ def get_status(ids_or_prefixes: list[str]) -> list[dict[str, Any]]:
     return results
 
 
+def stream_logs(
+    ids_or_prefixes: Union[str, list[str]],
+) -> Iterator[Tuple[str, str, str]]:
+    """Stream logs from jobs or invocations by their IDs or invocation IDs.
+
+    Args:
+        ids_or_prefixes: Single ID/prefix or list of job IDs or invocation IDs to stream logs from.
+                         Short prefixes are allowed, we would try to match the full ones from
+                         prefixes if no collisions are present.
+
+    Yields:
+        Tuple[str, str, str]: Tuples of (job_id, task_name, log_line) for each log line.
+            Empty lines are yielded as empty strings.
+
+    Raises:
+        ValueError: If the executor doesn't support log streaming.
+    """
+    db = ExecutionDB()
+
+    # Normalize to list for consistent processing
+    if isinstance(ids_or_prefixes, str):
+        ids_or_prefixes = [ids_or_prefixes]
+
+    # Collect all jobs from all IDs, grouped by executor
+    executor_to_jobs: Dict[str, Dict[str, JobData]] = {}
+    executor_to_invocations: Dict[str, list[str]] = {}
+
+    # TODO(agronskiy): refactor the `.`-checking job in all the functions.
+    for id_or_prefix in ids_or_prefixes:
+        # Determine if this is a job ID or invocation ID
+        if "." in id_or_prefix:
+            # This is a job ID
+            job_data = db.get_job(id_or_prefix)
+            if job_data is None:
+                continue
+
+            executor = job_data.executor
+            if executor not in executor_to_jobs:
+                executor_to_jobs[executor] = {}
+            executor_to_jobs[executor][id_or_prefix] = job_data
+        else:
+            # This is an invocation ID
+            jobs = db.get_jobs(id_or_prefix)
+            if not jobs:
+                continue
+
+            # Get the executor class from the first job
+            first_job_data = next(iter(jobs.values()))
+            executor = first_job_data.executor
+            if executor not in executor_to_invocations:
+                executor_to_invocations[executor] = []
+            executor_to_invocations[executor].append(id_or_prefix)
+
+    # Stream logs from each executor simultaneously
+    # For each executor, collect all job IDs and stream them together
+    for executor, jobs_dict in executor_to_jobs.items():
+        try:
+            executor_cls = get_executor(executor)
+        except ValueError:
+            continue
+
+        # For local executor with multiple jobs, pass list to stream simultaneously
+        # For other executors or single jobs, pass individual job IDs
+        if executor == "local" and len(jobs_dict) > 1:
+            # Pass all job IDs as a list to stream simultaneously
+            try:
+                yield from executor_cls.stream_logs(
+                    list(jobs_dict.keys()), executor_name=executor
+                )
+            except NotImplementedError:
+                raise ValueError(
+                    f"Log streaming is not yet implemented for executor '{executor}'"
+                )
+        else:
+            # Single job or non-local executor
+            for job_id in jobs_dict.keys():
+                try:
+                    yield from executor_cls.stream_logs(job_id, executor_name=executor)
+                except NotImplementedError:
+                    raise ValueError(
+                        f"Log streaming is not yet implemented for executor '{executor}'"
+                    )
+
+    # Stream logs from invocation IDs
+    for executor, invocation_ids in executor_to_invocations.items():
+        try:
+            executor_cls = get_executor(executor)
+        except ValueError:
+            continue
+
+        # Stream each invocation (each invocation already handles multiple jobs internally)
+        for invocation_id in invocation_ids:
+            try:
+                yield from executor_cls.stream_logs(
+                    invocation_id, executor_name=executor
+                )
+            except NotImplementedError:
+                raise ValueError(
+                    f"Log streaming is not yet implemented for executor '{executor}'"
+                )
+
+
 def list_all_invocations_summary() -> list[dict[str, Any]]:
     """Return a concise per-invocation summary from the exec DB.
 
@@ -378,6 +531,7 @@ def kill_job_or_invocation(id: str) -> list[dict[str, Any]]:
                 "data": {"error": f"Unexpected error: {str(e)}"},
             }
 
+    # TODO(agronskiy): refactor the `.`-checking job in all the functions.
     # Determine if this is a job ID or invocation ID
     if "." in id:
         # This is a job ID - kill single job
@@ -413,7 +567,7 @@ def kill_job_or_invocation(id: str) -> list[dict[str, Any]]:
 
 
 def export_results(
-    invocation_ids: Union[str, List[str]],
+    invocation_ids: Union[str, list[str]],
     dest: str = "local",
     config: dict[Any, Any] | None = None,
 ) -> dict:
@@ -442,7 +596,7 @@ def export_results(
             if "." in single_id:  # job_id
                 # Try reading config from artifacts working dir (auto-export on remote node)
                 cfg_file = None
-                for name in ("run_config.yml", "config.yml"):
+                for name in ("config.yml", "run_config.yml"):
                     p = Path(name)
                     if p.exists():
                         cfg_file = p

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.64}/src/nemo_evaluator_launcher/api/types.py

@@ -18,7 +18,7 @@
 This module defines data structures and helpers for configuration and type safety in the API layer.
 """
 
-import os
+import pathlib
 import warnings
 from dataclasses import dataclass
 from typing import cast
@@ -42,33 +42,40 @@ from nemo_evaluator_launcher.common.logging_utils import logger
 class RunConfig(DictConfig):
     @staticmethod
     def from_hydra(
-        config_name: str = "default",
-        config_dir: str | None = None,
-        hydra_overrides: list[str] = [],
-        dict_overrides: dict = {},
+        config: str | None = None,
+        hydra_overrides: list[str] | None = None,
+        dict_overrides: dict | None = None,
     ) -> "RunConfig":
         """Load configuration from Hydra and merge with dictionary overrides.
 
         Args:
-            config_name: Name of the Hydra configuration to load.
+            config: Optional full path to a config file (e.g. /path/to/my_config.yaml).
+                    If omitted, loads the internal default config from
+                    `nemo_evaluator_launcher.configs`.
             hydra_overrides: List of Hydra command-line style overrides.
             dict_overrides: Dictionary of configuration overrides to merge.
-            config_dir: Optional path to user config directory. If provided, Hydra will
-                       search in this directory first, then fall back to internal configs.
 
         Returns:
             RunConfig: Merged configuration object.
         """
-        overrides = hydra_overrides.copy()
+        overrides = list(hydra_overrides or [])
+        dict_overrides = dict_overrides or {}
+
+        resolved_config_path: str | None = None
+        config_name = "default"
+
         # Check if a GlobalHydra instance is already initialized and clear it
         if GlobalHydra.instance().is_initialized():
             GlobalHydra.instance().clear()
 
-        if config_dir:
-            # Convert relative path to absolute path if needed
-            if not os.path.isabs(config_dir):
-                config_dir = os.path.abspath(config_dir)
+        if config:
+            config_path = pathlib.Path(config).expanduser()
+            if not config_path.is_absolute():
+                config_path = (pathlib.Path.cwd() / config_path).resolve()
+            resolved_config_path = str(config_path)
 
+            config_dir = str(config_path.parent)
+            config_name = str(config_path.stem)
             hydra.initialize_config_dir(
                 config_dir=config_dir,
                 version_base=None,
@@ -90,7 +97,7 @@ class RunConfig(DictConfig):
         logger.debug(
             "Loaded run config from hydra",
             config_name=config_name,
-            config_dir=config_dir,
+            config=resolved_config_path,
             overrides=hydra_overrides,
             dict_overrides=dict_overrides,
             result=cfg,

nemo_evaluator_launcher-0.1.64/src/nemo_evaluator_launcher/cli/logs.py

@@ -0,0 +1,102 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Logs command for streaming logs from evaluation jobs."""
+
+import sys
+from dataclasses import dataclass
+from typing import Callable, Dict
+
+from simple_parsing import field
+
+import nemo_evaluator_launcher.common.printing_utils as pu
+from nemo_evaluator_launcher.api.functional import stream_logs
+from nemo_evaluator_launcher.common.execdb import ExecutionDB
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+
+@dataclass
+class Cmd:
+    """Logs command configuration."""
+
+    ids: list[str] = field(
+        default_factory=list,
+        positional=True,
+        help="Invocation IDs or job IDs (e.g., '15b9f667' or '15b9f667.0'). Multiple IDs can be provided.",
+    )
+
+    def execute(self) -> None:
+        """Execute the logs command to stream logs from jobs."""
+        if not self.ids:
+            logger.error("At least one ID is required")
+            sys.exit(1)
+
+        db = ExecutionDB()
+
+        # Validate all IDs exist
+        all_job_ids = []
+        for id_or_prefix in self.ids:
+            if "." in id_or_prefix:
+                # This is a job ID - get single job
+                job_data = db.get_job(id_or_prefix)
+                if job_data is None:
+                    logger.error(f"Job {id_or_prefix} not found")
+                    sys.exit(1)
+                all_job_ids.append(id_or_prefix)
+            else:
+                # This is an invocation ID - get all jobs
+                jobs = db.get_jobs(id_or_prefix)
+                if not jobs:
+                    logger.error(f"Invocation {id_or_prefix} not found")
+                    sys.exit(1)
+                all_job_ids.extend(jobs.keys())
+
+        # Build color mapping for job IDs
+        colors = [pu.red, pu.green, pu.yellow, pu.magenta, pu.cyan]
+        job_colors: Dict[str, Callable[[str], str]] = {}
+        color_index = 0
+
+        for job_id in all_job_ids:
+            job_colors[job_id] = colors[color_index % len(colors)]
+            color_index += 1
+
+        # Stream logs from executor
+        try:
+            log_stream = stream_logs(self.ids)
+            for job_id, task_name, log_line in log_stream:
+                # Extract short prefix: first 6 chars of invocation ID + job number
+                if "." in job_id:
+                    inv_id, job_num = job_id.split(".", 1)
+                    short_prefix = f"{inv_id[:6]}.{job_num}"
+                else:
+                    short_prefix = job_id[:6]
+                prefix = f"{short_prefix}:"
+                color_func = job_colors.get(job_id, pu.grey)
+                if log_line:
+                    print(f"{color_func(prefix)} {log_line}")
+                else:
+                    # Print empty lines without prefix
+                    print()
+
+        except ValueError:
+            # Handle case where executor doesn't support streaming
+            # Warning already logged by BaseExecutor.stream_logs
+            pass
+        except KeyboardInterrupt:
+            # Clean exit on Ctrl+C
+            pass
+        except Exception as e:
+            logger.error(f"Error streaming logs: {e}")
+            sys.exit(1)