nemo-evaluator-launcher 0.1.0rc2__py3-none-any.whl

nemo_evaluator_launcher/cli/main.py

@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Main CLI module using simple-parsing with subcommands."""
+
+from simple_parsing import ArgumentParser
+
+import nemo_evaluator_launcher.cli.export as export
+import nemo_evaluator_launcher.cli.kill as kill
+import nemo_evaluator_launcher.cli.ls_runs as ls_runs
+import nemo_evaluator_launcher.cli.ls_tasks as ls_tasks
+import nemo_evaluator_launcher.cli.run as run
+import nemo_evaluator_launcher.cli.status as status
+import nemo_evaluator_launcher.cli.version as version
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+VERSION_HELP = "Show version information"
+
+
+def create_parser() -> ArgumentParser:
+    """Create and configure the CLI argument parser with subcommands."""
+    parser = ArgumentParser()
+
+    # Add --version flag at the top level
+    parser.add_argument("--version", action="store_true", help=VERSION_HELP)
+
+    subparsers = parser.add_subparsers(dest="command", required=False)
+
+    # Version subcommand
+    version_parser = subparsers.add_parser(
+        "version",
+        help=VERSION_HELP,
+        description=VERSION_HELP,
+    )
+    version_parser.add_arguments(version.Cmd, dest="version")
+
+    # Run subcommand
+    run_parser = subparsers.add_parser(
+        "run", help="Run evaluation", description="Run evaluation"
+    )
+    run_parser.add_arguments(run.Cmd, dest="run")
+
+    # Status subcommand
+    status_parser = subparsers.add_parser(
+        "status", help="Check job status", description="Check job status"
+    )
+    status_parser.add_arguments(status.Cmd, dest="status")
+
+    # Kill subcommand
+    kill_parser = subparsers.add_parser(
+        "kill",
+        help="Kill a job or invocation",
+        description="Kill a job (e.g., aefc4819.0) or entire invocation (e.g., aefc4819) by its ID",
+    )
+    kill_parser.add_arguments(kill.Cmd, dest="kill")
+
+    # Ls subcommand (with nested subcommands)
+    ls_parser = subparsers.add_parser(
+        "ls", help="List resources", description="List tasks or runs"
+    )
+    ls_sub = ls_parser.add_subparsers(dest="ls_command", required=True)
+
+    # ls tasks
+    ls_tasks_parser = ls_sub.add_parser(
+        "tasks", help="List available tasks", description="List available tasks"
+    )
+    ls_tasks_parser.add_arguments(ls_tasks.Cmd, dest="tasks")
+
+    # ls runs (invocations summary)
+    ls_runs_parser = ls_sub.add_parser(
+        "runs",
+        help="List invocations (runs)",
+        description="Show a concise table of invocations from the exec DB",
+    )
+    ls_runs_parser.add_arguments(ls_runs.Cmd, dest="runs")
+
+    # Export subcommand
+    export_parser = subparsers.add_parser(
+        "export",
+        help="Export evaluation results",
+        description="Export evaluation results takes a List of invocation ids and a list of destinations(local, gitlab, wandb)",
+    )
+    export_parser.add_arguments(export.ExportCmd, dest="export")
+
+    return parser
+
+
+def main() -> None:
+    """Main CLI entry point with subcommands."""
+    parser = create_parser()
+    args = parser.parse_args()
+
+    # Handle --version flag
+    if hasattr(args, "version") and args.version:
+        version_cmd = version.Cmd()
+        version_cmd.execute()
+        return
+
+    # Handle case where no command is provided but --version wasn't used
+    if not hasattr(args, "command") or args.command is None:
+        parser.print_help()
+        return
+
+    logger.debug("Parsed arguments", args=args)
+    if args.command == "version":
+        args.version.execute()
+    elif args.command == "run":
+        args.run.execute()
+    elif args.command == "status":
+        args.status.execute()
+    elif args.command == "kill":
+        args.kill.execute()
+    elif args.command == "ls":
+        # Dispatch nested ls subcommands
+        if hasattr(args, "tasks"):
+            args.tasks.execute()
+        elif hasattr(args, "runs"):
+            args.runs.execute()
+    elif args.command == "export":
+        args.export.execute()
+
+
+if __name__ == "__main__":
+    main()

nemo_evaluator_launcher/cli/run.py

@@ -0,0 +1,135 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pathlib
+import time
+from dataclasses import dataclass
+
+import yaml
+from omegaconf import OmegaConf
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import RunConfig, run_eval
+
+
+@dataclass
+class Cmd:
+    """Run command parameters"""
+
+    config_name: str = field(
+        default="default",
+        alias=["-c", "--config-name"],
+        metadata={
+            "help": "Config name to use. Consult `nemo_evaluator_launcher.configs`"
+        },
+    )
+    config_dir: str | None = field(
+        default=None,
+        alias=["-d", "--config-dir"],
+        metadata={
+            "help": "Path to user config directory. If provided, searches here first, then falls back to internal configs."
+        },
+    )
+    run_config_file: str | None = field(
+        default=None,
+        alias=["-f", "--run-config-file"],
+        metadata={
+            "help": "Path to a run config file to load directly (bypasses Hydra config loading)."
+        },
+    )
+    override: list[str] = field(
+        default_factory=list,
+        action="append",
+        nargs="?",
+        alias=["-o"],
+        metadata={
+            "help": "Hydra override in the form some.param.path=value (pass multiple `-o` for multiple overrides).",
+        },
+    )
+    dry_run: bool = field(
+        default=False,
+        alias=["-n", "--dry-run"],
+        metadata={"help": "Do not run the evaluation, just print the config."},
+    )
+
+    def execute(self) -> None:
+        # Load configuration either from Hydra or from a run config file
+        if self.run_config_file:
+            # Validate that run config file is not used with other config options
+            if self.config_name != "default":
+                raise ValueError("Cannot use --run-config-file with --config-name")
+            if self.config_dir is not None:
+                raise ValueError("Cannot use --run-config-file with --config-dir")
+            if self.override:
+                raise ValueError("Cannot use --run-config-file with --override")
+
+            # Load from run config file
+            with open(self.run_config_file, "r") as f:
+                config_dict = yaml.safe_load(f)
+
+            # Create RunConfig from the loaded data
+            config = OmegaConf.create(config_dict)
+        else:
+            # Load the complete Hydra configuration
+            config = RunConfig.from_hydra(
+                config_name=self.config_name,
+                hydra_overrides=self.override,
+                config_dir=self.config_dir,
+            )
+
+        invocation_id = run_eval(config, self.dry_run)
+
+        # Save the complete configuration to the raw_configs directory
+        if not self.dry_run and invocation_id is not None:
+            # Create ~/.nemo-evaluator/run_configs directory
+            home_dir = pathlib.Path.home()
+            run_configs_dir = home_dir / ".nemo-evaluator" / "run_configs"
+            run_configs_dir.mkdir(parents=True, exist_ok=True)
+
+            # Convert DictConfig to dict and save as YAML
+            config_dict = OmegaConf.to_container(config, resolve=True)
+            config_yaml = yaml.dump(
+                config_dict, default_flow_style=False, sort_keys=False, indent=2
+            )
+
+            # Create config filename with invocation ID
+            config_filename = f"{invocation_id}_config.yml"
+            config_path = run_configs_dir / config_filename
+
+            # Save the complete Hydra configuration
+            with open(config_path, "w") as f:
+                f.write("# Complete configuration from nemo-evaluator-launcher\n")
+                f.write(
+                    f"# Generated at: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())}\n"
+                )
+                f.write(f"# Invocation ID: {invocation_id}\n")
+                f.write("#\n")
+                f.write("# This is the complete raw configuration\n")
+                f.write("#\n")
+                f.write("# To rerun this exact configuration:\n")
+                f.write(
+                    f"# nemo-evaluator-launcher run --run-config-file {config_path}\n"
+                )
+                f.write("#\n")
+                f.write(config_yaml)
+
+            print(f"Complete run config saved to: {config_path}")
+
+        if invocation_id is not None:
+            print(f"to check status: nemo-evaluator-launcher status {invocation_id}")
+            print(f"to kill all jobs: nemo-evaluator-launcher kill {invocation_id}")
+            print(
+                f"to kill individual jobs: nemo-evaluator-launcher kill <job_id> (e.g., {invocation_id}.0)"
+            )

nemo_evaluator_launcher/cli/status.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+from dataclasses import dataclass
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.api.functional import get_status
+
+
+@dataclass
+class Cmd:
+    """Status command configuration."""
+
+    job_ids: list[str] = field(
+        default_factory=list,
+        positional=True,
+    )
+    json: bool = field(
+        default=False,
+        action="store_true",
+        help="Print output as JSON instead of table format",
+    )
+
+    def execute(self) -> None:
+        res = get_status(self.job_ids)
+        if self.json:
+            # Remove progress field from JSON output as it's a WIP feature
+            filtered_res = []
+            for job in res:
+                job_copy = job.copy()
+                job_copy.pop("progress", None)
+                filtered_res.append(job_copy)
+            print(json.dumps(filtered_res, indent=2))
+        else:
+            self._print_table(res)
+
+    def _print_table(self, jobs: list[dict]) -> None:
+        """Print job status as a table."""
+        if not jobs:
+            print("No jobs found.")
+            return
+
+        # Define executor-specific mappings
+        executor_headers = {
+            "slurm_job_id": "Slurm Job ID",
+            "lepton_job_name": "Lepton Job Name",
+            "pipeline_id": "Pipeline ID",
+            "container": "Container",
+        }
+
+        # Determine executor type and headers
+        first_data = jobs[0].get("data", {}) if jobs else {}
+        executor_key = next((k for k in executor_headers if k in first_data), None)
+        info_header = executor_headers.get(executor_key, "Executor Info")
+        headers = ["Job ID", "Status", info_header, "Location"]
+
+        # Build rows
+        rows = []
+        for job in jobs:
+            data = job.get("data", {})
+
+            # Extract executor info
+            executor_info = str(data.get(executor_key, "")) if executor_key else ""
+
+            # Extract location
+            if executor_key == "slurm_job_id":
+                path = data.get("remote_rundir_path", "")
+                location = (
+                    "<output_dir>/" + "/".join(path.split("/")[-2:]) if path else ""
+                )
+            elif executor_key == "lepton_job_name":
+                location = data.get("endpoint_name") or "shared-endpoint"
+            elif executor_key == "pipeline_id":
+                location = data.get("pipeline_web_url")
+            elif executor_key == "container":
+                path = data.get("output_dir", "")
+                location = path.split("/")[-1] if path else ""
+            else:
+                location = ""
+
+            rows.append(
+                [
+                    job.get("job_id", ""),
+                    job.get("status", ""),
+                    # job.get("progress", ""), temporarily disabled as this is a WIP feature
+                    executor_info,
+                    location,
+                ]
+            )
+
+        # Calculate column widths and print
+        widths = [
+            max(len(str(headers[i])), max(len(str(row[i])) for row in rows))
+            for i in range(len(headers))
+        ]
+
+        header_row = " | ".join(
+            headers[i].ljust(widths[i]) for i in range(len(headers))
+        )
+        print(header_row)
+        print("-" * len(header_row))
+
+        for row in rows:
+            print(" | ".join(str(row[i]).ljust(widths[i]) for i in range(len(row))))

nemo_evaluator_launcher/cli/version.py

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Version command for nemo-evaluator-launcher."""
+
+import importlib
+from dataclasses import dataclass
+
+from nemo_evaluator_launcher import __package_name__, __version__
+
+
+@dataclass
+class Cmd:
+    """Show version information for nemo-evaluator-launcher and internal packages."""
+
+    def execute(self) -> None:
+        """Execute the version command."""
+        print(f"{__package_name__}: {__version__}")
+
+        # Check for internal package
+        try:
+            internal_module = importlib.import_module(
+                "nemo_evaluator_launcher_internal"
+            )
+            # Try to get version from internal package
+            try:
+                internal_version = getattr(internal_module, "__version__", None)
+                if internal_version:
+                    print(f"nemo-evaluator-launcher-internal: {internal_version}")
+                else:
+                    print(
+                        "nemo-evaluator-launcher-internal: available (version unknown)"
+                    )
+            except Exception:
+                print("nemo-evaluator-launcher-internal: available (version unknown)")
+        except ImportError:
+            # Internal package not available - this is expected in many cases
+            pass
+        except Exception as e:
+            print(f"nemo-evaluator-launcher-internal: error loading ({e})")

nemo_evaluator_launcher/common/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Common utilities and configurations for nemo-evaluator-launcher."""

nemo_evaluator_launcher/common/execdb.py

@@ -0,0 +1,189 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Execution database module for tracking job executions."""
+
+import json
+import pathlib
+import secrets
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List, Optional
+
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+# Configuration constants
+EXEC_DB_DIR = pathlib.Path.home() / ".nemo-evaluator" / "exec-db"
+EXEC_DB_FILE = EXEC_DB_DIR / "exec.v1.jsonl"
+
+
+def generate_invocation_id() -> str:
+    """Generate a unique invocation ID as an 8-digit hex string."""
+    return secrets.token_hex(4)
+
+
+def generate_job_id(invocation_id: str, index: int) -> str:
+    """Generate a job ID as <invocation_id>.<n>.
+
+    Args:
+        invocation_id: The invocation group ID (8-digit hex).
+        index: The job index (0-based integer).
+    Returns:
+        The job ID string.
+    """
+    return f"{invocation_id}.{index}"
+
+
+@dataclass
+class JobData:
+    """Data structure for job execution information.
+
+    Attributes:
+        invocation_id: 8-digit hex string.
+        job_id: <invocation_id>.<n> string.
+        timestamp: Unix timestamp when the job was created.
+        executor: Name of the executor that handled this job.
+        data: Additional job-specific data as a dictionary.
+        config: Configuration used to setup a job.
+    """
+
+    invocation_id: str
+    job_id: str
+    timestamp: float
+    executor: str
+    data: Dict[str, Any]
+    config: Optional[Dict[str, Any]] = None
+
+
+class ExecutionDB:
+    """Singleton class for managing execution database with invocation and job hierarchy."""
+
+    _instance: Optional["ExecutionDB"] = None
+    _jobs: Dict[str, JobData] = {}  # job_id -> JobData
+    _invocations: Dict[str, List[str]] = {}  # invocation_id -> list of job_ids
+
+    def __new__(cls) -> "ExecutionDB":
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self) -> None:
+        if not hasattr(self, "_initialized"):
+            self._ensure_db_dir()
+            self._load_existing_jobs()
+            self._initialized = True
+
+    def _ensure_db_dir(self) -> None:
+        EXEC_DB_DIR.mkdir(parents=True, exist_ok=True)
+
+    def _load_existing_jobs(self) -> None:
+        if not EXEC_DB_FILE.exists():
+            return
+        try:
+            with open(EXEC_DB_FILE, "r") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        record = json.loads(line)
+                        invocation_id = record.get("invocation_id")
+                        job_id = record.get("job_id")
+                        executor = record.get("executor")
+                        data = record.get("data", {})
+                        config = record.get("config", {})
+                        timestamp = record.get("timestamp", 0.0)
+                        if invocation_id and job_id and executor:
+                            job_data = JobData(
+                                invocation_id=invocation_id,
+                                job_id=job_id,
+                                timestamp=timestamp,
+                                executor=executor,
+                                data=data,
+                                config=config,
+                            )
+                            self._jobs[job_id] = job_data
+                            if invocation_id not in self._invocations:
+                                self._invocations[invocation_id] = []
+                            self._invocations[invocation_id].append(job_id)
+                    except json.JSONDecodeError as e:
+                        logger.warning("Failed to parse JSONL line", error=str(e))
+        except OSError as e:
+            logger.warning("Failed to load existing jobs", error=str(e))
+
+    def write_job(self, job: JobData) -> None:
+        if job.job_id:
+            self._jobs[job.job_id] = job
+        if job.invocation_id not in self._invocations:
+            self._invocations[job.invocation_id] = []
+        if job.job_id and job.job_id not in self._invocations[job.invocation_id]:
+            self._invocations[job.invocation_id].append(job.job_id)
+        record = asdict(job)
+        try:
+            with open(EXEC_DB_FILE, "a") as f:
+                f.write(json.dumps(record) + "\n")
+            logger.info(
+                "Job written to execution database",
+                invocation_id=job.invocation_id,
+                job_id=job.job_id,
+                executor=job.executor,
+            )
+        except OSError as e:
+            logger.error(
+                "Failed to write job to database",
+                invocation_id=job.invocation_id,
+                job_id=job.job_id,
+                error=str(e),
+            )
+            raise
+
+    def get_job(self, job_id: str) -> Optional[JobData]:
+        return self._jobs.get(job_id)
+
+    def get_jobs(self, invocation_id: str) -> Dict[str, JobData]:
+        job_ids = self._invocations.get(invocation_id, [])
+        return {
+            job_id: self._jobs[job_id] for job_id in job_ids if job_id in self._jobs
+        }
+
+    def get_invocation_jobs(self, invocation_id: str) -> List[str]:
+        return self._invocations.get(invocation_id, [])
+
+    def get_all_jobs(self) -> Dict[str, JobData]:
+        """Return a copy of all jobs in the execution DB."""
+        return dict(self._jobs)
+
+
+def write_job(job: JobData) -> None:
+    db = ExecutionDB()
+    db.write_job(job)
+
+
+def get_job(job_id: str) -> Optional[JobData]:
+    db = ExecutionDB()
+    return db.get_job(job_id)
+
+
+def get_jobs(invocation_id: str) -> Dict[str, JobData]:
+    db = ExecutionDB()
+    return db.get_jobs(invocation_id)
+
+
+def get_all_jobs() -> Dict[str, JobData]:
+    db = ExecutionDB()
+    return db.get_all_jobs()
+
+
+# Ensure all the paths
+_DB = ExecutionDB()