nemo-evaluator-launcher 0.1.21__tar.gz → 0.1.44__tar.gz

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.21
+Version: 0.1.44
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com
@@ -458,7 +458,7 @@ License:
               OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
               SOFTWARE.
         
-Project-URL: homepage, https://github.com/NVIDIA-NeMo/Eval
+Project-URL: homepage, https://github.com/NVIDIA-NeMo/Evaluator
 Project-URL: repository, https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-launcher
 Keywords: deep learning,evaluations,machine learning,gpu,NLP,pytorch,torch
 Requires-Python: <3.14,>=3.10
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
 Provides-Extra: wandb
 Requires-Dist: wandb>=0.15.0; extra == "wandb"
 Provides-Extra: gsheets
-Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
+Requires-Dist: gspread>=5.0.0; extra == "gsheets"
 Provides-Extra: exporters
 Requires-Dist: mlflow; extra == "exporters"
 Requires-Dist: wandb; extra == "exporters"

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.44}/pyproject.toml

@@ -33,20 +33,21 @@ keywords = [
 
 [project.urls]
 # BEGIN(if-changed): check package_info.py
-homepage = "https://github.com/NVIDIA-NeMo/Eval"
+homepage = "https://github.com/NVIDIA-NeMo/Evaluator"
 repository = "https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-launcher"
 # END(if-changed)
 
 [project.optional-dependencies]
 mlflow = ["mlflow>=2.8.0"]
 wandb = ["wandb>=0.15.0"]
-gsheets = ["gsheets>=0.1.0"]
+gsheets = ["gspread>=5.0.0"]
 exporters = ["mlflow", "wandb", "gsheets"]
 all = ["mlflow", "wandb", "gsheets"]
 
 [project.scripts]
 nemo-evaluator-launcher = "nemo_evaluator_launcher.cli.main:main"
 nv-eval = "nemo_evaluator_launcher.cli.main:main"
+nel = "nemo_evaluator_launcher.cli.main:main"
 
 [dependency-groups]
 dev = [
@@ -75,6 +76,7 @@ where = ["src"]
   "resources/**/*",
   "configs/**/*",
   "executors/**/*.sh",
+  "executors/**/*.template",
 ]
 
 [tool.setuptools.dynamic]

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.44}/src/nemo_evaluator_launcher/api/functional.py

@@ -19,7 +19,7 @@ This module provides the main functional entry points for running evaluations, q
 """
 
 from pathlib import Path
-from typing import Any, List, Optional, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import yaml
 from omegaconf import DictConfig, OmegaConf
@@ -116,6 +116,7 @@ def get_status(ids_or_prefixes: list[str]) -> list[dict[str, Any]]:
     db = ExecutionDB()
     results: List[dict[str, Any]] = []
 
+    # TODO(agronskiy): refactor the `.`-checking job in all the functions.
     for id_or_prefix in ids_or_prefixes:
         # If id looks like an invocation_id (no dot), get all jobs for it
         if "." not in id_or_prefix:
@@ -259,6 +260,108 @@ def get_status(ids_or_prefixes: list[str]) -> list[dict[str, Any]]:
     return results
 
 
+def stream_logs(
+    ids_or_prefixes: Union[str, list[str]],
+) -> Iterator[Tuple[str, str, str]]:
+    """Stream logs from jobs or invocations by their IDs or invocation IDs.
+
+    Args:
+        ids_or_prefixes: Single ID/prefix or list of job IDs or invocation IDs to stream logs from.
+                         Short prefixes are allowed, we would try to match the full ones from
+                         prefixes if no collisions are present.
+
+    Yields:
+        Tuple[str, str, str]: Tuples of (job_id, task_name, log_line) for each log line.
+            Empty lines are yielded as empty strings.
+
+    Raises:
+        ValueError: If the executor doesn't support log streaming.
+    """
+    db = ExecutionDB()
+
+    # Normalize to list for consistent processing
+    if isinstance(ids_or_prefixes, str):
+        ids_or_prefixes = [ids_or_prefixes]
+
+    # Collect all jobs from all IDs, grouped by executor
+    executor_to_jobs: Dict[str, Dict[str, JobData]] = {}
+    executor_to_invocations: Dict[str, List[str]] = {}
+
+    # TODO(agronskiy): refactor the `.`-checking job in all the functions.
+    for id_or_prefix in ids_or_prefixes:
+        # Determine if this is a job ID or invocation ID
+        if "." in id_or_prefix:
+            # This is a job ID
+            job_data = db.get_job(id_or_prefix)
+            if job_data is None:
+                continue
+
+            executor = job_data.executor
+            if executor not in executor_to_jobs:
+                executor_to_jobs[executor] = {}
+            executor_to_jobs[executor][id_or_prefix] = job_data
+        else:
+            # This is an invocation ID
+            jobs = db.get_jobs(id_or_prefix)
+            if not jobs:
+                continue
+
+            # Get the executor class from the first job
+            first_job_data = next(iter(jobs.values()))
+            executor = first_job_data.executor
+            if executor not in executor_to_invocations:
+                executor_to_invocations[executor] = []
+            executor_to_invocations[executor].append(id_or_prefix)
+
+    # Stream logs from each executor simultaneously
+    # For each executor, collect all job IDs and stream them together
+    for executor, jobs_dict in executor_to_jobs.items():
+        try:
+            executor_cls = get_executor(executor)
+        except ValueError:
+            continue
+
+        # For local executor with multiple jobs, pass list to stream simultaneously
+        # For other executors or single jobs, pass individual job IDs
+        if executor == "local" and len(jobs_dict) > 1:
+            # Pass all job IDs as a list to stream simultaneously
+            try:
+                yield from executor_cls.stream_logs(
+                    list(jobs_dict.keys()), executor_name=executor
+                )
+            except NotImplementedError:
+                raise ValueError(
+                    f"Log streaming is not yet implemented for executor '{executor}'"
+                )
+        else:
+            # Single job or non-local executor
+            for job_id in jobs_dict.keys():
+                try:
+                    yield from executor_cls.stream_logs(job_id, executor_name=executor)
+                except NotImplementedError:
+                    raise ValueError(
+                        f"Log streaming is not yet implemented for executor '{executor}'"
+                    )
+
+    # Stream logs from invocation IDs
+    for executor, invocation_ids in executor_to_invocations.items():
+        try:
+            executor_cls = get_executor(executor)
+        except ValueError:
+            continue
+
+        # Stream each invocation (each invocation already handles multiple jobs internally)
+        for invocation_id in invocation_ids:
+            try:
+                yield from executor_cls.stream_logs(
+                    invocation_id, executor_name=executor
+                )
+            except NotImplementedError:
+                raise ValueError(
+                    f"Log streaming is not yet implemented for executor '{executor}'"
+                )
+
+
 def list_all_invocations_summary() -> list[dict[str, Any]]:
     """Return a concise per-invocation summary from the exec DB.
 
@@ -378,6 +481,7 @@ def kill_job_or_invocation(id: str) -> list[dict[str, Any]]:
                 "data": {"error": f"Unexpected error: {str(e)}"},
             }
 
+    # TODO(agronskiy): refactor the `.`-checking job in all the functions.
     # Determine if this is a job ID or invocation ID
     if "." in id:
         # This is a job ID - kill single job
@@ -442,7 +546,7 @@ def export_results(
             if "." in single_id:  # job_id
                 # Try reading config from artifacts working dir (auto-export on remote node)
                 cfg_file = None
-                for name in ("run_config.yml", "config.yml"):
+                for name in ("config.yml", "run_config.yml"):
                     p = Path(name)
                     if p.exists():
                         cfg_file = p

nemo_evaluator_launcher-0.1.44/src/nemo_evaluator_launcher/cli/logs.py

@@ -0,0 +1,102 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Logs command for streaming logs from evaluation jobs."""
+
+import sys
+from dataclasses import dataclass
+from typing import Callable, Dict
+
+from simple_parsing import field
+
+import nemo_evaluator_launcher.common.printing_utils as pu
+from nemo_evaluator_launcher.api.functional import stream_logs
+from nemo_evaluator_launcher.common.execdb import ExecutionDB
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+
+@dataclass
+class Cmd:
+    """Logs command configuration."""
+
+    ids: list[str] = field(
+        default_factory=list,
+        positional=True,
+        help="Invocation IDs or job IDs (e.g., '15b9f667' or '15b9f667.0'). Multiple IDs can be provided.",
+    )
+
+    def execute(self) -> None:
+        """Execute the logs command to stream logs from jobs."""
+        if not self.ids:
+            logger.error("At least one ID is required")
+            sys.exit(1)
+
+        db = ExecutionDB()
+
+        # Validate all IDs exist
+        all_job_ids = []
+        for id_or_prefix in self.ids:
+            if "." in id_or_prefix:
+                # This is a job ID - get single job
+                job_data = db.get_job(id_or_prefix)
+                if job_data is None:
+                    logger.error(f"Job {id_or_prefix} not found")
+                    sys.exit(1)
+                all_job_ids.append(id_or_prefix)
+            else:
+                # This is an invocation ID - get all jobs
+                jobs = db.get_jobs(id_or_prefix)
+                if not jobs:
+                    logger.error(f"Invocation {id_or_prefix} not found")
+                    sys.exit(1)
+                all_job_ids.extend(jobs.keys())
+
+        # Build color mapping for job IDs
+        colors = [pu.red, pu.green, pu.yellow, pu.magenta, pu.cyan]
+        job_colors: Dict[str, Callable[[str], str]] = {}
+        color_index = 0
+
+        for job_id in all_job_ids:
+            job_colors[job_id] = colors[color_index % len(colors)]
+            color_index += 1
+
+        # Stream logs from executor
+        try:
+            log_stream = stream_logs(self.ids)
+            for job_id, task_name, log_line in log_stream:
+                # Extract short prefix: first 6 chars of invocation ID + job number
+                if "." in job_id:
+                    inv_id, job_num = job_id.split(".", 1)
+                    short_prefix = f"{inv_id[:6]}.{job_num}"
+                else:
+                    short_prefix = job_id[:6]
+                prefix = f"{short_prefix}:"
+                color_func = job_colors.get(job_id, pu.grey)
+                if log_line:
+                    print(f"{color_func(prefix)} {log_line}")
+                else:
+                    # Print empty lines without prefix
+                    print()
+
+        except ValueError:
+            # Handle case where executor doesn't support streaming
+            # Warning already logged by BaseExecutor.stream_logs
+            pass
+        except KeyboardInterrupt:
+            # Clean exit on Ctrl+C
+            pass
+        except Exception as e:
+            logger.error(f"Error streaming logs: {e}")
+            sys.exit(1)

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.44}/src/nemo_evaluator_launcher/cli/main.py

@@ -22,6 +22,7 @@ from simple_parsing import ArgumentParser
 import nemo_evaluator_launcher.cli.export as export
 import nemo_evaluator_launcher.cli.info as info
 import nemo_evaluator_launcher.cli.kill as kill
+import nemo_evaluator_launcher.cli.logs as logs
 import nemo_evaluator_launcher.cli.ls_runs as ls_runs
 import nemo_evaluator_launcher.cli.ls_tasks as ls_tasks
 import nemo_evaluator_launcher.cli.run as run
@@ -42,6 +43,7 @@ def is_verbose_enabled(args) -> bool:
     subcommands = [
         "run",
         "status",
+        "logs",
         "info",
         "kill",
         "tasks_alias",
@@ -106,6 +108,14 @@ def create_parser() -> ArgumentParser:
     )
     status_parser.add_arguments(status.Cmd, dest="status")
 
+    # Logs subcommand
+    logs_parser = subparsers.add_parser(
+        "logs",
+        help="Stream logs from evaluation jobs",
+        description="Stream logs from evaluation jobs by invocation ID or job ID",
+    )
+    logs_parser.add_arguments(logs.Cmd, dest="logs")
+
     # Kill subcommand
     kill_parser = subparsers.add_parser(
         "kill",
@@ -204,6 +214,8 @@ def main() -> None:
         args.run.execute()
     elif args.command == "status":
         args.status.execute()
+    elif args.command == "logs":
+        args.logs.execute()
     elif args.command == "kill":
         args.kill.execute()
     elif args.command == "ls":

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.44}/src/nemo_evaluator_launcher/cli/run.py

@@ -16,6 +16,7 @@
 import pathlib
 import time
 from dataclasses import dataclass
+from typing import Literal
 
 from simple_parsing import field
 
@@ -26,6 +27,7 @@ from nemo_evaluator_launcher.common.printing_utils import (
     green,
     magenta,
     red,
+    yellow,
 )
 
 
@@ -33,6 +35,13 @@ from nemo_evaluator_launcher.common.printing_utils import (
 class Cmd:
     """Run command parameters"""
 
+    config: str | None = field(
+        default=None,
+        alias=["--config"],
+        metadata={
+            "help": "Full path to config file. Uses Hydra by default (--config-mode=hydra). Use --config-mode=raw to load directly (bypasses Hydra)."
+        },
+    )
     config_name: str = field(
         default="default",
         alias=["-c", "--config-name"],
@@ -47,11 +56,11 @@ class Cmd:
             "help": "Path to user config directory. If provided, searches here first, then falls back to internal configs."
         },
     )
-    run_config_file: str | None = field(
-        default=None,
-        alias=["-f", "--run-config-file"],
+    config_mode: Literal["hydra", "raw"] = field(
+        default="hydra",
+        alias=["--config-mode"],
         metadata={
-            "help": "Path to a run config file to load directly (bypasses Hydra config loading)."
+            "help": "Config loading mode: 'hydra' (default) uses Hydra config system, 'raw' loads config file directly bypassing Hydra."
         },
     )
     override: list[str] = field(
@@ -83,28 +92,59 @@ class Cmd:
 
         from nemo_evaluator_launcher.api.functional import RunConfig, run_eval
 
-        # Load configuration either from Hydra or from a run config file
-        if self.run_config_file:
-            # Validate that run config file is not used with other config options
+        # Validate config_mode value
+        if self.config_mode not in ["hydra", "raw"]:
+            raise ValueError(
+                f"Invalid --config-mode value: {self.config_mode}. Must be 'hydra' or 'raw'."
+            )
+
+        # Validate that raw mode requires --config
+        if self.config_mode == "raw" and self.config is None:
+            raise ValueError(
+                "--config-mode=raw requires --config to be specified. Raw mode loads config files directly."
+            )
+
+        # Load configuration either from Hydra or directly from a config file
+        if self.config_mode == "raw" and self.config:
+            # Validate that raw config loading is not used with other config options
             if self.config_name != "default":
-                raise ValueError("Cannot use --run-config-file with --config-name")
+                raise ValueError(
+                    "Cannot use --config-mode=raw with --config-name. Raw mode only works with --config."
+                )
             if self.config_dir is not None:
-                raise ValueError("Cannot use --run-config-file with --config-dir")
+                raise ValueError(
+                    "Cannot use --config-mode=raw with --config-dir. Raw mode only works with --config."
+                )
             if self.override:
-                raise ValueError("Cannot use --run-config-file with --override")
+                raise ValueError(
+                    "Cannot use --config-mode=raw with --override. Raw mode only works with --config."
+                )
 
-            # Load from run config file
-            with open(self.run_config_file, "r") as f:
+            # Load from config file directly (bypass Hydra)
+            with open(self.config, "r") as f:
                 config_dict = yaml.safe_load(f)
 
             # Create RunConfig from the loaded data
             config = OmegaConf.create(config_dict)
         else:
+            # Handle --config parameter: split path into config_dir and config_name for Hydra
+            if self.config:
+                if self.config_name != "default":
+                    raise ValueError("Cannot use --config with --config-name")
+                if self.config_dir is not None:
+                    raise ValueError("Cannot use --config with --config-dir")
+                config_path = pathlib.Path(self.config)
+                config_dir = str(config_path.parent)
+                config_name = str(config_path.stem)
+            else:
+                config_dir = self.config_dir
+                config_name = self.config_name
+
             # Load the complete Hydra configuration
             config = RunConfig.from_hydra(
-                config_name=self.config_name,
+                config_dir=config_dir,
+                config_name=config_name,
                 hydra_overrides=self.override,
-                config_dir=self.config_dir,
             )
 
         try:
@@ -150,7 +190,7 @@ class Cmd:
                 f.write("#\n")
                 f.write("# To rerun this exact configuration:\n")
                 f.write(
-                    f"# nemo-evaluator-launcher run --run-config-file {config_path}\n"
+                    f"# nemo-evaluator-launcher run --config {config_path} --config-mode=raw\n"
                 )
                 f.write("#\n")
                 f.write(config_yaml)
@@ -164,6 +204,10 @@ class Cmd:
                 bold(cyan("To check status: "))
                 + f"nemo-evaluator-launcher status {invocation_id}"
             )
+            print(
+                bold(cyan("To view job info: "))
+                + f"nemo-evaluator-launcher info {invocation_id}"
+            )
             print(
                 bold(cyan("To kill all jobs: "))
                 + f"nemo-evaluator-launcher kill {invocation_id}"
@@ -198,3 +242,17 @@ class Cmd:
                     )
                 )
             )
+
+        # Warn if both config_dir and config_name are provided (and config_name is not default)
+        if (
+            self.config is None
+            and self.config_dir is not None
+            and self.config_name != "default"
+        ):
+            joint_path = pathlib.Path(self.config_dir) / f"{self.config_name}.yaml"
+            print(
+                yellow(
+                    f"Warning: Using --config-dir and --config-name together is deprecated. "
+                    f"Please use --config {joint_path} instead."
+                )
+            )

{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.44}/src/nemo_evaluator_launcher/cli/version.py

@@ -19,6 +19,29 @@ import importlib
 from dataclasses import dataclass
 
 from nemo_evaluator_launcher import __package_name__, __version__
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+
+def get_versions() -> dict:
+    internal_module_name = "nemo_evaluator_launcher_internal"
+    res = {__package_name__: __version__}
+    # Check for internal package
+    try:
+        internal_module = importlib.import_module(internal_module_name)
+        # Try to get version from internal package
+        internal_version = getattr(internal_module, "__version__", None)
+        if internal_version:
+            res[internal_module_name] = internal_version
+        else:
+            res[internal_module_name] = "available (version unknown)"
+    except ImportError:
+        # Internal package not available - this is expected in many cases
+        pass
+    except Exception as e:
+        logger.error(f"nemo_evaluator_launcher_internal: error loading ({e})")
+        raise
+
+    return res
 
 
 @dataclass
@@ -27,26 +50,6 @@ class Cmd:
 
     def execute(self) -> None:
         """Execute the version command."""
-        print(f"{__package_name__}: {__version__}")
-
-        # Check for internal package
-        try:
-            internal_module = importlib.import_module(
-                "nemo_evaluator_launcher_internal"
-            )
-            # Try to get version from internal package
-            try:
-                internal_version = getattr(internal_module, "__version__", None)
-                if internal_version:
-                    print(f"nemo-evaluator-launcher-internal: {internal_version}")
-                else:
-                    print(
-                        "nemo-evaluator-launcher-internal: available (version unknown)"
-                    )
-            except Exception:
-                print("nemo-evaluator-launcher-internal: available (version unknown)")
-        except ImportError:
-            # Internal package not available - this is expected in many cases
-            pass
-        except Exception as e:
-            print(f"nemo-evaluator-launcher-internal: error loading ({e})")
+        res = get_versions()
+        for package, version in res.items():
+            print(f"{package}: {version}")