nemo-evaluator-launcher 0.1.0rc2__py3-none-any.whl

nemo_evaluator_launcher/api/types.py

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Type definitions for the nemo-evaluator-launcher public API.
+
+This module defines data structures and helpers for configuration and type safety in the API layer.
+"""
+
+import os
+from dataclasses import dataclass
+from typing import cast
+
+import hydra
+from hydra.core.global_hydra import GlobalHydra
+from omegaconf import DictConfig, OmegaConf
+
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+
+@dataclass
+class RunConfig(DictConfig):
+    @staticmethod
+    def from_hydra(
+        config_name: str = "default",
+        config_dir: str | None = None,
+        hydra_overrides: list[str] = [],
+        dict_overrides: dict = {},
+    ) -> "RunConfig":
+        """Load configuration from Hydra and merge with dictionary overrides.
+
+        Args:
+            config_name: Name of the Hydra configuration to load.
+            hydra_overrides: List of Hydra command-line style overrides.
+            dict_overrides: Dictionary of configuration overrides to merge.
+            config_dir: Optional path to user config directory. If provided, Hydra will
+                       search in this directory first, then fall back to internal configs.
+
+        Returns:
+            RunConfig: Merged configuration object.
+        """
+        overrides = hydra_overrides.copy()
+        # Check if a GlobalHydra instance is already initialized and clear it
+        if GlobalHydra.instance().is_initialized():
+            GlobalHydra.instance().clear()
+
+        if config_dir:
+            # Convert relative path to absolute path if needed
+            if not os.path.isabs(config_dir):
+                config_dir = os.path.abspath(config_dir)
+
+            hydra.initialize_config_dir(
+                config_dir=config_dir,
+                version_base=None,
+            )
+        else:
+            hydra.initialize_config_module(
+                config_module="nemo_evaluator_launcher.configs",
+                version_base=None,
+            )
+        overrides = overrides + [
+            "hydra.searchpath=[pkg://nemo_evaluator_launcher.configs,pkg://nemo_evaluator_launcher_internal.configs]"
+        ]
+        cfg = hydra.compose(config_name=config_name, overrides=overrides)
+
+        # Merge dict_overrides if provided
+        if dict_overrides:
+            cfg = OmegaConf.merge(cfg, dict_overrides)
+
+        logger.debug(
+            "Loaded run config from hydra",
+            config_name=config_name,
+            config_dir=config_dir,
+            overrides=hydra_overrides,
+            dict_overrides=dict_overrides,
+            result=cfg,
+        )
+        return cast("RunConfig", cfg)

nemo_evaluator_launcher/api/utils.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Configuration validation utilities for nemo-evaluator-launcher.
+
+This module provides helper functions to validate configuration objects.
+"""

nemo_evaluator_launcher/cli/__init__.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

nemo_evaluator_launcher/cli/export.py

@@ -0,0 +1,148 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Export evaluation results to specified target."""
+
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import export_results
+
+
+@dataclass
+class ExportCmd:
+    """Export evaluation results."""
+
+    # Short usage examples will show up in -h as the class docstring:
+    # Examples:
+    #   nemo-evaluator-launcher export 8abcd123 --dest local --format json -o .
+    #   nemo-evaluator-launcher export 8abcd123.0 9ef01234 --dest local --format csv -o results/ -fname processed_results.csv
+    #   nemo-evaluator-launcher export 8abcd123 --dest jet
+
+    invocation_ids: List[str] = field(
+        positional=True,
+        help="IDs to export (space-separated). Accepts invocation IDs (xxxxxxxx) and job IDs (xxxxxxxx.n); mixture of both allowed.",
+    )
+    dest: str = field(
+        default="local",
+        alias=["--dest"],
+        choices=["local", "wandb", "mlflow", "gsheets", "jet"],
+        help="Export destination.",
+    )
+    output_dir: Optional[str] = field(
+        default=".",
+        alias=["--output-dir", "-o"],
+        help="Output directory (default: current directory).",
+    )
+    output_filename: Optional[str] = field(
+        default=None,
+        alias=["--output-filename", "-fname"],
+        help="Summary filename (default: processed_results.json/csv based on --format).",
+    )
+    format: Optional[str] = field(
+        default=None,
+        alias=["--format"],
+        choices=["json", "csv"],
+        help="Summary format for --dest local. Omit to only copy artifacts.",
+    )
+    copy_logs: bool = field(
+        default=False,
+        alias=["--copy-logs"],
+        help="Include logs when copying locally (default: False).",
+    )
+    log_metrics: List[str] = field(
+        default_factory=list,
+        alias=["--log-metrics"],
+        help="Filter metrics by name (repeatable). Examples: score, f1, mmlu_score_micro.",
+    )
+    only_required: bool = field(
+        default=True,
+        alias=["--only-required"],
+        help="Copy only required+optional artifacts (default: True). Set to False to copy all available artifacts.",
+    )
+
+    def execute(self) -> None:
+        """Execute export."""
+        config: dict[str, Any] = {
+            "copy_logs": self.copy_logs,
+            "only_required": self.only_required,
+        }
+
+        # Output handling
+        if self.output_dir:
+            config["output_dir"] = self.output_dir
+        if self.output_filename:
+            config["output_filename"] = self.output_filename
+
+        # Format and filters
+        if self.format:
+            config["format"] = self.format
+        if self.log_metrics:
+            config["log_metrics"] = self.log_metrics
+
+        if self.format and self.dest != "local":
+            print(
+                "Note: --format is only used by --dest local. It will be ignored for other destinations."
+            )
+
+        # Execute
+        print(
+            f"Exporting {len(self.invocation_ids)} {'invocations' if len(self.invocation_ids) > 1 else 'invocation'} to {self.dest}..."
+        )
+
+        result = export_results(self.invocation_ids, self.dest, config)
+
+        if not result["success"]:
+            print(f"Export failed: {result.get('error', 'Unknown error')}")
+            return
+
+        # Success path
+        if len(self.invocation_ids) == 1:
+            # Single invocation
+            invocation_id = self.invocation_ids[0]
+            print(f"Export completed for {invocation_id}")
+
+            for job_id, job_result in result["jobs"].items():
+                if job_result.get("success"):
+                    print(f"  {job_id}: {job_result.get('message', '')}")
+                    metadata = job_result.get("metadata", {})
+                    if metadata.get("run_url"):
+                        print(f"    URL: {metadata['run_url']}")
+                    if metadata.get("summary_path"):
+                        print(f"    Summary: {metadata['summary_path']}")
+                else:
+                    print(f"  {job_id} failed: {job_result.get('message', '')}")
+        else:
+            # Multiple invocations
+            metadata = result.get("metadata", {})
+            print(
+                f"Export completed: {metadata.get('successful_invocations', 0)}/{metadata.get('total_invocations', 0)} successful"
+            )
+
+            # Show summary path if available
+            if metadata.get("summary_path"):
+                print(f"Summary: {metadata['summary_path']}")
+
+            # Show per-invocation status
+            for invocation_id, inv_result in result["invocations"].items():
+                if inv_result.get("success"):
+                    job_count = len(inv_result.get("jobs", {}))
+                    print(f"  {invocation_id}: {job_count} jobs")
+                else:
+                    print(
+                        f"  {invocation_id}: failed, {inv_result.get('error', 'Unknown error')}"
+                    )

nemo_evaluator_launcher/cli/info.py

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import datetime as _dt
+import sys
+from dataclasses import dataclass
+from typing import Optional
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import (
+    get_invocation_benchmarks,
+    list_all_invocations_summary,
+)
+
+
+@dataclass
+class Cmd:
+    """Show a concise invocations table from the exec DB."""
+
+    limit: Optional[int] = field(default=None, alias=["--limit"], help="Max rows")
+    executor: Optional[str] = field(
+        default=None, alias=["--executor"], help="Filter by executor"
+    )
+    since: Optional[str] = field(
+        default=None,
+        alias=["--since"],
+        help="Filter by ISO date/time (e.g., 2025-08-20 or 2025-08-20T12:00:00)",
+    )
+
+    def execute(self) -> None:
+        rows = list_all_invocations_summary()
+
+        # Apply filters
+        if self.executor:
+            rows = [
+                r
+                for r in rows
+                if (r.get("executor") or "").lower() == self.executor.lower()
+            ]
+
+        if self.since:
+            try:
+                # Parse with optional time
+                if "T" in self.since:
+                    since_ts = _dt.datetime.fromisoformat(self.since).timestamp()
+                else:
+                    since_ts = _dt.datetime.fromisoformat(
+                        self.since + "T00:00:00"
+                    ).timestamp()
+                rows = [r for r in rows if (r.get("earliest_job_ts") or 0) >= since_ts]
+            except Exception:
+                print(
+                    f"Invalid --since value: {self.since}. Use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS",
+                    file=sys.stderr,
+                )
+                sys.exit(2)
+
+        if self.limit is not None and self.limit >= 0:
+            rows = rows[: self.limit]
+
+        # Print table (always include benchmarks column)
+        header = [
+            "invocation_id",
+            "earliest_job_ts",
+            "num_jobs",
+            "executor",
+            "benchmarks",
+        ]
+        # Format ts as ISO without microseconds
+        table_rows = []
+        for r in rows:
+            ts = r.get("earliest_job_ts", 0) or 0
+            try:
+                ts_iso = (
+                    _dt.datetime.fromtimestamp(ts).replace(microsecond=0).isoformat()
+                )
+            except Exception:
+                ts_iso = ""
+            inv = r.get("invocation_id", "")
+            try:
+                bmarks = get_invocation_benchmarks(inv)
+                bmarks_cell = ",".join(bmarks) if bmarks else "unknown"
+            except Exception:
+                bmarks_cell = "unknown"
+            table_rows.append(
+                [
+                    str(inv),
+                    ts_iso,
+                    str(r.get("num_jobs", 0)),
+                    str(r.get("executor", "")),
+                    bmarks_cell,
+                ]
+            )
+
+        widths = [len(h) for h in header]
+        for tr in table_rows:
+            for i, cell in enumerate(tr):
+                if len(cell) > widths[i]:
+                    widths[i] = len(cell)
+        fmt = "  ".join([f"{{:<{w}}}" for w in widths])
+        print(fmt.format(*header))
+        print("  ".join(["-" * w for w in widths]))
+        for tr in table_rows:
+            print(fmt.format(*tr))

nemo_evaluator_launcher/cli/kill.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+from dataclasses import dataclass
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import kill_job_or_invocation
+
+
+@dataclass
+class Cmd:
+    """Kill command configuration."""
+
+    id: str = field(
+        positional=True,
+        metadata={
+            "help": "Job ID (e.g., aefc4819.0) or invocation ID (e.g., aefc4819) to kill"
+        },
+    )
+
+    def execute(self) -> None:
+        """Execute the kill command."""
+        result = kill_job_or_invocation(self.id)
+        # Output as JSON
+        print(json.dumps(result, indent=2))

nemo_evaluator_launcher/cli/ls_runs.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import datetime as _dt
+import sys
+from dataclasses import dataclass
+from typing import Optional
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import (
+    get_invocation_benchmarks,
+    list_all_invocations_summary,
+)
+
+
+@dataclass
+class Cmd:
+    """List invocations (runs) from the exec DB as a table."""
+
+    limit: Optional[int] = field(default=None, alias=["--limit"], help="Max rows")
+    executor: Optional[str] = field(
+        default=None, alias=["--executor"], help="Filter by executor"
+    )
+    since: Optional[str] = field(
+        default=None,
+        alias=["--since"],
+        help="Filter by ISO date/time (e.g., 2025-08-20 or 2025-08-20T12:00:00)",
+    )
+
+    def execute(self) -> None:
+        rows = list_all_invocations_summary()
+
+        if self.executor:
+            rows = [
+                r
+                for r in rows
+                if (r.get("executor") or "").lower() == self.executor.lower()
+            ]
+
+        if self.since:
+            try:
+                if "T" in self.since:
+                    since_ts = _dt.datetime.fromisoformat(self.since).timestamp()
+                else:
+                    since_ts = _dt.datetime.fromisoformat(
+                        self.since + "T00:00:00"
+                    ).timestamp()
+                rows = [r for r in rows if (r.get("earliest_job_ts") or 0) >= since_ts]
+            except Exception:
+                print(
+                    f"Invalid --since value: {self.since}. Use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS",
+                    file=sys.stderr,
+                )
+                sys.exit(2)
+
+        if self.limit is not None and self.limit >= 0:
+            rows = rows[: self.limit]
+
+        header = [
+            "invocation_id",
+            "earliest_job_ts",
+            "num_jobs",
+            "executor",
+            "benchmarks",
+        ]
+        table_rows = []
+        for r in rows:
+            ts = r.get("earliest_job_ts", 0) or 0
+            try:
+                ts_iso = (
+                    _dt.datetime.fromtimestamp(ts).replace(microsecond=0).isoformat()
+                )
+            except Exception:
+                ts_iso = ""
+            inv = r.get("invocation_id", "")
+            try:
+                bmarks = get_invocation_benchmarks(inv)
+                bmarks_cell = ",".join(bmarks) if bmarks else "unknown"
+            except Exception:
+                bmarks_cell = "unknown"
+            table_rows.append(
+                [
+                    str(inv),
+                    ts_iso,
+                    str(r.get("num_jobs", 0)),
+                    str(r.get("executor", "")),
+                    bmarks_cell,
+                ]
+            )
+
+        widths = [len(h) for h in header]
+        for tr in table_rows:
+            for i, cell in enumerate(tr):
+                if len(cell) > widths[i]:
+                    widths[i] = len(cell)
+        fmt = "  ".join([f"{{:<{w}}}" for w in widths])
+        print(fmt.format(*header))
+        print("  ".join(["-" * w for w in widths]))
+        for tr in table_rows:
+            print(fmt.format(*tr))

nemo_evaluator_launcher/cli/ls_tasks.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+from dataclasses import dataclass
+
+from nemo_evaluator_launcher.api.functional import get_tasks_list
+
+
+@dataclass
+class Cmd:
+    """List command configuration."""
+
+    def execute(self) -> None:
+        # TODO(dfridman): modify `get_tasks_list` to return a list of dicts in the first place
+        data = get_tasks_list()
+        headers = ["task", "endpoint_type", "harness", "container"]
+        supported_benchmarks = []
+        for task_data in data:
+            assert len(task_data) == len(headers)
+            supported_benchmarks.append(dict(zip(headers, task_data)))
+        print(json.dumps(supported_benchmarks, indent=2))