nemo-evaluator-launcher 0.1.15__tar.gz → 0.1.17__tar.gz

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.15
+Version: 0.1.17
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

nemo_evaluator_launcher-0.1.17/src/nemo_evaluator_launcher/cli/debug.py

@@ -0,0 +1,405 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Debugging helper functionalities for nemo-evaluator-launcher."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from simple_parsing import field
+
+from nemo_evaluator_launcher.cli.export import ExportCmd
+from nemo_evaluator_launcher.cli.version import Cmd as VersionCmd
+from nemo_evaluator_launcher.common.execdb import EXEC_DB_FILE, ExecutionDB, JobData
+from nemo_evaluator_launcher.common.logging_utils import logger
+from nemo_evaluator_launcher.exporters.local import LocalExporter
+from nemo_evaluator_launcher.exporters.utils import get_task_name
+
+# Local exporter helper to copy logs and artifacts
+_EXPORT_HELPER = LocalExporter({})
+
+
+@dataclass
+class DebugCmd(ExportCmd):
+    """Debugging functionalities for nemo-evaluator-launcher.
+
+    Examples:
+      nemo-evaluator-launcher debug <inv>                 # Full debug info
+      nemo-evaluator-launcher debug <inv> --config        # Show stored job config (YAML)
+      nemo-evaluator-launcher debug <inv> --artifacts     # Show artifact locations
+      nemo-evaluator-launcher debug <inv> --logs          # Show log locations
+      nemo-evaluator-launcher debug <inv> --copy-logs <path>       # Copy logs (default: current dir)
+      nemo-evaluator-launcher debug <inv> --copy-artifacts <path>   # Copy artifacts (default: current dir)
+
+    Notes:
+      - Supports invocation IDs and job IDs
+      - Shows local or remote paths depending on executor (local/slurm/lepton)
+    """
+
+    # local exporter destination defaults to local
+    dest: str = field(default="local", init=False)
+
+    # debug modes
+    config: bool = field(default=False, help="Show job configuration")
+    artifacts: bool = field(default=False, help="Show artifact locations")
+    logs: bool = field(default=False, help="Show log locations")
+
+    # copy operations
+    copy_logs: Optional[str] = field(
+        default=None,
+        alias=["--copy-logs"],
+        nargs="?",
+        help="Copy logs to local directory (default: current dir)",
+    )
+    copy_artifacts: Optional[str] = field(
+        default=None,
+        alias=["--copy-artifacts"],
+        nargs="?",
+        help="Copy artifacts to local directory (default: current dir)",
+    )
+
+    def execute(self) -> None:
+        # show version
+        VersionCmd().execute()
+
+        logger.info("Debug command started", invocation_ids=self.invocation_ids)
+
+        if not self.invocation_ids:
+            logger.error("No invocation IDs provided")
+            raise ValueError("No job or invocation IDs provided.")
+
+        jobs = self._resolve_jobs()
+        logger.info(
+            "Resolved jobs",
+            total_ids=len(self.invocation_ids),
+            valid_jobs=len(jobs),
+            job_ids=[jid for jid, _ in jobs],
+        )
+
+        if not jobs:
+            logger.info(
+                "No valid jobs found (jobs may have been deleted or IDs may be incorrect)."
+            )
+            print(
+                "No valid jobs found (jobs may have been deletedd or IDs may be incorrect)."
+            )
+            return
+
+        if self.config:
+            logger.info("Showing job configuration", job_count=len(jobs))
+            self._show_config_info(jobs)
+        elif self.logs:
+            logger.info("Showing job logs locations", job_count=len(jobs))
+            self._show_logs_info(jobs)
+        elif self.artifacts:
+            logger.info("Showing artifacts locations", job_count=len(jobs))
+            self._show_artifacts_info(jobs)
+        elif self.copy_logs is not None:
+            dest = self.copy_logs or "."
+            if not self.copy_logs:
+                print(
+                    "No destination provided for --copy-logs; defaulting to current dir"
+                )
+            logger.info(
+                "Copying logs to local directory", dest_dir=dest, job_count=len(jobs)
+            )
+            self._copy_logs(jobs, dest)
+        elif self.copy_artifacts is not None:
+            dest = self.copy_artifacts or "."
+            if not self.copy_artifacts:
+                print(
+                    "No destination provided for --copy-artifacts; defaulting to current dir)"
+                )
+            logger.info(
+                "Copying artifacts to local directory",
+                dest_dir=dest,
+                job_count=len(jobs),
+            )
+            self._copy_artifacts(jobs, dest)
+        else:
+            logger.info(
+                "Job metadata details",
+                invocation_id=jobs[0][1].invocation_id if jobs else None,
+                jobs=len(jobs),
+            )
+            self._show_invocation_debug_info(jobs)
+
+    def _resolve_jobs(self) -> List[Tuple[str, JobData]]:
+        """Resolve jobs from ExecDB using IDs (job IDs and/or invocation IDs)."""
+        db = ExecutionDB()
+        found: list[tuple[str, JobData]] = []
+        for id_or_prefix in self.invocation_ids:
+            if "." in id_or_prefix:
+                jd = db.get_job(id_or_prefix)
+                if jd:
+                    found.append((jd.job_id, jd))
+            else:
+                for jid, jd in db.get_jobs(id_or_prefix).items():
+                    found.append((jid, jd))
+        # deduplicate and stable sort
+        seen: set[str] = set()
+        uniq: list[tuple[str, JobData]] = []
+        for jid, jd in found:
+            if jid not in seen:
+                seen.add(jid)
+                uniq.append((jid, jd))
+        return sorted(uniq, key=lambda p: p[0])
+
+    def _show_invocation_debug_info(self, jobs: List[Tuple[str, JobData]]) -> None:
+        inv = jobs[0][1].invocation_id if jobs else None
+        logger.info("Debug information", jobs=len(jobs), invocation=inv)
+        print(
+            f"Debug information for {len(jobs)} job(s){f' under invocation {inv}' if inv else ''}:\n"
+        )
+
+        for job_id, job_data in jobs:
+            self._show_job_debug_info(job_id, job_data)
+            print()
+
+        # footer hint: where to find more metadata
+        print(
+            "For more details about this run, inspect the Execution DB under your home dir:"
+        )
+        print(f"Path: {EXEC_DB_FILE}")
+        if inv:
+            print(f"├── Lookup key: invocation_id={inv}")
+
+        # Next steps hint
+        print("\nNext steps:")
+        print("  - Use --logs to show log locations.")
+        print("  - Use --artifacts to show artifact locations.")
+        print("  - Use --config to show stored job configuration (YAML).")
+        print("  - Use --copy-logs [DIR] to copy logs to a local directory.")
+        print("  - Use --copy-artifacts [DIR] to copy artifacts to a local directory.")
+
+    def _show_job_debug_info(self, job_id: str, job_data: JobData) -> None:
+        logger.info("Job", job_id=job_id)
+        print(f"Job {job_id}")
+
+        # metadata
+        try:
+            when = datetime.fromtimestamp(job_data.timestamp).isoformat(
+                timespec="seconds"
+            )
+        except Exception:
+            when = str(job_data.timestamp)
+        logger.info("Executor", job_id=job_id, executor=job_data.executor)
+        logger.info("Created", job_id=job_id, created=when)
+        print(f"├── Executor: {job_data.executor}")
+        print(f"├── Created: {when}")
+
+        task_name = get_task_name(job_data)
+        if task_name:
+            logger.info("Task", job_id=job_id, name=task_name)
+            print(f"├── Task: {task_name}")
+
+        # locations via exporter helper
+        paths = _EXPORT_HELPER.get_job_paths(job_data)
+
+        # Artifacts
+        if paths.get("storage_type") == "remote_ssh":
+            artifacts_path = f"{paths['username']}@{paths['hostname']}:{paths['remote_path']}/artifacts"
+            logger.info("Artifacts", job_id=job_id, path=artifacts_path, remote=True)
+            print(f"├── Artifacts: {artifacts_path} (remote)")
+        else:
+            ap = paths.get("artifacts_dir")
+            if ap:
+                exists = self._check_path_exists(paths, "artifacts")
+                logger.info(
+                    "Artifacts", job_id=job_id, path=str(ap), exists_indicator=exists
+                )
+                print(f"├── Artifacts: {ap} {exists} (local)")
+
+        # Logs
+        if paths.get("storage_type") == "remote_ssh":
+            logs_path = (
+                f"{paths['username']}@{paths['hostname']}:{paths['remote_path']}/logs"
+            )
+            logger.info("Logs", job_id=job_id, path=logs_path, remote=True)
+            print(f"├── Logs: {logs_path} (remote)")
+        else:
+            lp = paths.get("logs_dir")
+            if lp:
+                exists = self._check_path_exists(paths, "logs")
+                logger.info(
+                    "Logs", job_id=job_id, path=str(lp), exists_indicator=exists
+                )
+                print(f"├── Logs: {lp} {exists} (local)")
+
+        # executor-specific
+        d = job_data.data or {}
+        cfg_exec_type = ((job_data.config or {}).get("execution") or {}).get("type")
+        exec_type = (job_data.executor or cfg_exec_type or "").lower()
+
+        if exec_type == "slurm":
+            sj = d.get("slurm_job_id")
+            if sj:
+                print(f"├── Slurm Job ID: {sj}")
+        elif exec_type == "gitlab":
+            pid = d.get("pipeline_id")
+            if pid:
+                print(f"├── Pipeline ID: {pid}")
+        elif exec_type == "lepton":
+            jn = d.get("lepton_job_name")
+            if jn:
+                print(f"├── Lepton Job: {jn}")
+            en = d.get("endpoint_name")
+            if en:
+                print(f"├── Endpoint: {en}")
+            eu = d.get("endpoint_url")
+            if eu:
+                print(f"├── Endpoint URL: {eu}")
+        # local and others: paths already displayed above; no extra fields needed
+
+    def _show_logs_info(self, jobs: List[Tuple[str, JobData]]) -> None:
+        logger.info("Log locations")
+        print("Log locations:\n")
+        for job_id, job_data in jobs:
+            paths = _EXPORT_HELPER.get_job_paths(job_data)
+            if paths.get("storage_type") == "remote_ssh":
+                logs_path = f"ssh://{paths['username']}@{paths['hostname']}{paths['remote_path']}/logs"
+                logger.info("Logs", job_id=job_id, path=logs_path, remote=True)
+                print(f"{job_id}: {logs_path} (remote)")
+            else:
+                lp = paths.get("logs_dir")
+                if lp:
+                    exists = self._check_path_exists(paths, "logs")
+                    logger.info(
+                        "Logs", job_id=job_id, path=str(lp), exists_indicator=exists
+                    )
+                    print(f"{job_id}: {lp} {exists} (local)")
+
+    def _show_artifacts_info(self, jobs: List[Tuple[str, JobData]]) -> None:
+        logger.info("Artifact locations")
+        print("Artifact locations:\n")
+        for job_id, job_data in jobs:
+            paths = _EXPORT_HELPER.get_job_paths(job_data)
+            if paths.get("storage_type") == "remote_ssh":
+                artifacts_path = f"ssh://{paths['username']}@{paths['hostname']}{paths['remote_path']}/artifacts"
+                logger.info(
+                    "Artifacts", job_id=job_id, path=artifacts_path, remote=True
+                )
+                print(f"{job_id}: {artifacts_path} (remote)")
+            else:
+                ap = paths.get("artifacts_dir")
+                if ap:
+                    exists = self._check_path_exists(paths, "artifacts")
+                    logger.info(
+                        "Artifacts",
+                        job_id=job_id,
+                        path=str(ap),
+                        exists_indicator=exists,
+                    )
+                    print(f"{job_id}: {ap} {exists} (local)")
+
+    def _show_config_info(self, jobs: List[Tuple[str, JobData]]) -> None:
+        for job_id, job_data in jobs:
+            logger.info("Configuration for job", job_id=job_id)
+            print(f"Configuration for {job_id}:")
+            if job_data.config:
+                import yaml
+
+                config_yaml = yaml.dump(
+                    job_data.config, default_flow_style=False, indent=2
+                )
+                logger.info("Configuration YAML", job_id=job_id, config=config_yaml)
+                print(config_yaml)
+            else:
+                logger.info("No configuration stored for this job", job_id=job_id)
+                print("  No configuration stored for this job.")
+            print()
+
+    def _copy_logs(self, jobs: List[Tuple[str, JobData]], dest_dir: str) -> None:
+        """Copy logs using export functionality."""
+        self._copy_content(jobs, dest_dir, copy_logs=True, copy_artifacts=False)
+
+    def _copy_artifacts(self, jobs: List[Tuple[str, JobData]], dest_dir: str) -> None:
+        """Copy artifacts using export functionality."""
+        self._copy_content(jobs, dest_dir, copy_logs=False, copy_artifacts=True)
+
+    def _copy_content(
+        self,
+        jobs: List[Tuple[str, JobData]],
+        dest_dir: str,
+        copy_logs: bool,
+        copy_artifacts: bool,
+    ) -> None:
+        logger.debug(
+            "Preparing export call",
+            dest_dir=dest_dir,
+            copy_logs=copy_logs,
+            copy_artifacts=copy_artifacts,
+            job_ids=[jid for jid, _ in jobs],
+        )
+
+        from nemo_evaluator_launcher.api.functional import export_results
+
+        config = {
+            "output_dir": dest_dir,
+            "only_required": True,
+            "copy_logs": bool(copy_logs) and not bool(copy_artifacts),
+            "copy_artifacts": bool(copy_artifacts) and not bool(copy_logs),
+        }
+        # skip artifact validation
+        if copy_logs and not copy_artifacts:
+            config["skip_validation"] = True
+
+        job_ids = [job_id for job_id, _ in jobs]
+        kind = "logs" if copy_logs else "artifacts"
+        logger.info(
+            "Copying content", kind=kind, job_count=len(job_ids), dest_dir=dest_dir
+        )
+        print(f"Copying {kind} for {len(job_ids)} job(s) to {dest_dir}...")
+
+        result = export_results(job_ids, "local", config)
+        logger.debug("Export API call completed", success=result.get("success"))
+
+        if result.get("success"):
+            logger.info(
+                "Content copy completed successfully",
+                dest_dir=dest_dir,
+                job_count=len(jobs),
+            )
+            if "jobs" in result:
+                for jid, job_result in result["jobs"].items():
+                    if job_result.get("success"):
+                        print(f"{jid}: Success")
+                    else:
+                        print(
+                            f"{jid}: Failed - {job_result.get('message', 'Unknown error')}"
+                        )
+        else:
+            err = result.get("error", "Unknown error")
+            logger.warning("Content copy failed", error=err, dest_dir=dest_dir)
+            print(f"Failed to copy {kind}: {err}")
+
+    def _check_path_exists(self, paths: Dict[str, Any], path_type: str) -> str:
+        """Check if a path exists and return indicator."""
+        try:
+            if paths.get("storage_type") == "remote_ssh":
+                # For remote paths, we can't easily check existence
+                return "(remote)"
+            elif path_type == "logs" and "logs_dir" in paths:
+                logs_dir = Path(paths["logs_dir"])
+                return "(exists)" if logs_dir.exists() else "(not found)"
+            elif path_type == "artifacts" and "artifacts_dir" in paths:
+                artifacts_dir = Path(paths["artifacts_dir"])
+                return "(exists)" if artifacts_dir.exists() else "(not found)"
+        except Exception:
+            pass
+        return ""

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher/cli/ls_runs.py

@@ -20,6 +20,8 @@ from typing import Optional
 
 from simple_parsing import field
 
+from nemo_evaluator_launcher.common.logging_utils import logger
+
 
 @dataclass
 class Cmd:
@@ -27,12 +29,16 @@ class Cmd:
 
     limit: Optional[int] = field(default=None, alias=["--limit"], help="Max rows")
     executor: Optional[str] = field(
-        default=None, alias=["--executor"], help="Filter by executor"
+        default=None,
+        alias=["--executor"],
+        help="Filter by executor",
     )
+    # TODO(agronskiy): think about if we can propagate a `--status` filter into here.
     since: Optional[str] = field(
         default=None,
         alias=["--since"],
-        help="Filter by ISO date/time (e.g., 2025-08-20 or 2025-08-20T12:00:00)",
+        help="Filter by either ISO date/time (e.g., 2025-08-20 or 2025-08-20T12:00:00) or "
+        "an interval into the past, e.g. `1d` or `3h`; formally `{N}[d|h]`.",
     )
 
     def execute(self) -> None:
@@ -53,7 +59,22 @@ class Cmd:
 
         if self.since:
             try:
-                if "T" in self.since:
+                # Check if it's a relative time format like "1d" or "3h"
+                if self.since.lower().endswith("d") and len(self.since) > 1:
+                    days = int(self.since[:-1])
+                    if days < 0:
+                        raise ValueError("Days should be non-negative")
+                    since_ts = (
+                        _dt.datetime.now() - _dt.timedelta(days=days)
+                    ).timestamp()
+                elif self.since.lower().endswith("h") and len(self.since) > 1:
+                    hours = int(self.since[:-1])
+                    if hours < 0:
+                        raise ValueError("Hours should be non-negative")
+                    since_ts = (
+                        _dt.datetime.now() - _dt.timedelta(hours=hours)
+                    ).timestamp()
+                elif "T" in self.since:
                     since_ts = _dt.datetime.fromisoformat(self.since).timestamp()
                 else:
                     since_ts = _dt.datetime.fromisoformat(
@@ -61,9 +82,8 @@ class Cmd:
                     ).timestamp()
                 rows = [r for r in rows if (r.get("earliest_job_ts") or 0) >= since_ts]
             except Exception:
-                print(
-                    f"Invalid --since value: {self.since}. Use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS",
-                    file=sys.stderr,
+                logger.fatal(
+                    f"Invalid --since value: {self.since}. Use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or N[d|h] for N days|hours."
                 )
                 sys.exit(2)
 

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher/cli/main.py

@@ -19,6 +19,7 @@ import os
 
 from simple_parsing import ArgumentParser
 
+import nemo_evaluator_launcher.cli.debug as debug
 import nemo_evaluator_launcher.cli.export as export
 import nemo_evaluator_launcher.cli.kill as kill
 import nemo_evaluator_launcher.cli.ls_runs as ls_runs
@@ -38,7 +39,16 @@ def is_verbose_enabled(args) -> bool:
         return True
 
     # Check subcommand verbose flags
-    subcommands = ["run", "status", "kill", "tasks_alias", "tasks", "runs", "export"]
+    subcommands = [
+        "run",
+        "status",
+        "kill",
+        "tasks_alias",
+        "tasks",
+        "runs",
+        "export",
+        "debug",
+    ]
     for subcmd in subcommands:
         if hasattr(args, subcmd) and hasattr(getattr(args, subcmd), "verbose"):
             if getattr(getattr(args, subcmd), "verbose"):
@@ -153,6 +163,17 @@ def create_parser() -> ArgumentParser:
     )
     export_parser.add_arguments(export.ExportCmd, dest="export")
 
+    # Debug helper subcommand
+    debug_parser = subparsers.add_parser(
+        "debug",
+        help="Display evaluation job information",
+        description="Debug helper functionalities for nemo-evaluator-launcher",
+    )
+    debug_parser.add_argument(
+        "-v", "--verbose", action="store_true", help="Enable verbose logging"
+    )
+    debug_parser.add_arguments(debug.DebugCmd, dest="debug")
+
     return parser
 
 
@@ -197,6 +218,8 @@ def main() -> None:
             args.runs.execute()
     elif args.command == "export":
         args.export.execute()
+    elif args.command == "debug":
+        args.debug.execute()
 
 
 if __name__ == "__main__":

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher/cli/run.py

@@ -168,3 +168,7 @@ class Cmd:
             for idx, task in enumerate(tasks):
                 job_id = f"{invocation_id}.{idx}"
                 print(f"  nemo-evaluator-launcher kill {job_id}  # {task.name}")
+            print(
+                "to print all jobs: nemo-evaluator-launcher ls runs"
+                "\n  (--since 1d or --since 6h for time span, see --help)"
+            )

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher/executors/lepton/executor.py

@@ -78,9 +78,32 @@ class LeptonExecutor(BaseExecutor):
                 "LeptonExecutor supports deployment types: 'vllm', 'sglang', 'nim', 'none'"
             )
 
+        # Load tasks mapping
+        tasks_mapping = load_tasks_mapping()
+        job_ids = []
+        lepton_job_names = []
+        endpoint_names = []  # Track multiple endpoints
+        db = ExecutionDB()
+
         # Generate invocation ID
         invocation_id = generate_invocation_id()
 
+        # DRY-RUN mode
+        if dry_run:
+            output_dir = Path(cfg.execution.output_dir).absolute() / invocation_id
+            output_dir.mkdir(parents=True, exist_ok=True)
+
+            # Validate configuration
+            _dry_run_lepton(cfg, tasks_mapping, invocation_id=invocation_id)
+
+            if cfg.deployment.type == "none":
+                print("Using existing endpoint (deployment: none)")
+                print("using shared endpoint")
+            else:
+                print(f"with endpoint type '{cfg.deployment.type}'")
+
+            return invocation_id
+
         # For deployment: none, we use the existing endpoint for all tasks
         if cfg.deployment.type == "none":
             print("📌 Using existing endpoint (deployment: none)")
@@ -88,13 +111,6 @@ class LeptonExecutor(BaseExecutor):
             print(f"✅ Using shared endpoint: {shared_endpoint_url}")
 
         try:
-            # Load tasks mapping
-            tasks_mapping = load_tasks_mapping()
-            job_ids = []
-            lepton_job_names = []
-            endpoint_names = []  # Track multiple endpoints
-            db = ExecutionDB()
-
             # Create local directory for outputs
             output_dir = Path(cfg.execution.output_dir).absolute() / invocation_id
             output_dir.mkdir(parents=True, exist_ok=True)
@@ -139,8 +155,13 @@ class LeptonExecutor(BaseExecutor):
                     task_index = str(idx)
                     endpoint_name = f"{cfg.deployment.type}-{short_task_name}-{task_index}-{short_invocation}"
 
-                    # Ensure we don't exceed 36 character limit
                     if len(endpoint_name) > 36:
+                        logger.info(
+                            "Lepton endpoint name will be deployed under name {task_name}",
+                            task_name=task.name,
+                            original=endpoint_name,
+                            limit=36,
+                        )
                         # Truncate task name further if needed
                         max_task_len = (
                             36
@@ -151,7 +172,19 @@ class LeptonExecutor(BaseExecutor):
                         )  # 3 hyphens
                         short_task_name = sanitized_task_name[:max_task_len]
                         endpoint_name = f"{cfg.deployment.type}-{short_task_name}-{task_index}-{short_invocation}"
+                        logger.info(
+                            "Lepton endpoint name is auto-generated",
+                            task_name=task.name,
+                            original=endpoint_name,
+                            truncated=endpoint_name,
+                            limit=36,
+                        )
 
+                    logger.info(
+                        "Lepton endpoint name (auto-generated)",
+                        task_name=task.name,
+                        endpoint_name=endpoint_name,
+                    )
                     endpoint_names.append(endpoint_name)
                     endpoint_creation_tasks.append((idx, task, endpoint_name))
 
@@ -298,20 +331,6 @@ class LeptonExecutor(BaseExecutor):
                     f"✅ All {len(cfg.evaluation.tasks)} endpoints created successfully!"
                 )
 
-            if dry_run:
-                print("🔍 DRY RUN: Lepton job configurations prepared")
-                print(f"   - Tasks: {len(cfg.evaluation.tasks)}")
-                for idx, task in enumerate(cfg.evaluation.tasks):
-                    if cfg.deployment.type == "none":
-                        print(f"   - Task {idx}: {task.name} using shared endpoint")
-                    else:
-                        print(
-                            f"   - Task {idx}: {task.name} with endpoint {endpoint_names[idx]}"
-                        )
-                print(f"   - Output directory: {output_dir}")
-                print("\nTo submit jobs, run the executor without --dry-run")
-                return invocation_id
-
             # ================================================================
             # JOB SUBMISSION (Sequential, as before)
             # ================================================================
@@ -334,8 +353,18 @@ class LeptonExecutor(BaseExecutor):
                 max_base_length = 36 - 1 - len(suffix)  # -1 for the hyphen
                 if len(base_job_name) > max_base_length:
                     base_job_name = base_job_name[:max_base_length]
+                    logger.info(
+                        "Lepton job auto-generated name",
+                        task_name=task.name,
+                        job_name=f"{base_job_name}-{suffix}",
+                    )
 
                 lepton_job_name = f"{base_job_name}-{suffix}"
+                logger.info(
+                    "Lepton job name (auto-generated)",
+                    task_name=task.name,
+                    job_name=lepton_job_name,
+                )
                 job_ids.append(job_id)
                 lepton_job_names.append(lepton_job_name)
 
@@ -773,6 +802,82 @@ exit 0
     return script
 
 
+def _dry_run_lepton(
+    cfg: DictConfig, tasks_mapping: dict, invocation_id: str | None = None
+) -> None:
+    print("DRY RUN: Lepton job configurations prepared")
+    try:
+        # validate tasks
+        for task in cfg.evaluation.tasks:
+            get_task_from_mapping(task.name, tasks_mapping)
+
+        # nice-to-have checks (existing endpoint URL or endpoints mapping)
+        if getattr(cfg.deployment, "type", None) == "none":
+            tgt = getattr(cfg, "target", {})
+            api = (
+                tgt.get("api_endpoint")
+                if isinstance(tgt, dict)
+                else getattr(tgt, "api_endpoint", None)
+            ) or {}
+            url = api.get("url") if isinstance(api, dict) else getattr(api, "url", None)
+            if not url or not str(url).strip():
+                raise ValueError(
+                    "target.api_endpoint.url must be set when deployment.type == 'none'"
+                )
+        else:
+            endpoints_cfg = getattr(cfg.deployment, "endpoints", {}) or {}
+            for task in cfg.evaluation.tasks:
+                td = get_task_from_mapping(task.name, tasks_mapping)
+                etype = td.get("endpoint_type")
+                if etype not in endpoints_cfg:
+                    raise ValueError(
+                        f"deployment.endpoints missing path for endpoint_type '{etype}' (task '{task.name}')"
+                    )
+                path = endpoints_cfg.get(etype)
+                if not isinstance(path, str) or not path.startswith("/"):
+                    raise ValueError(
+                        f"deployment.endpoints['{etype}'] must be a non-empty path starting with '/'"
+                    )
+
+        # lepton env var presence (reference-level)
+        tasks_cfg = getattr(cfg.execution, "lepton_platform", {}).get("tasks", {}) or {}
+        lepton_env_vars = tasks_cfg.get("env_vars", {}) or {}
+        api_key_name = getattr(
+            getattr(cfg, "target", {}).get("api_endpoint", {}), "api_key_name", None
+        )
+        for task in cfg.evaluation.tasks:
+            td = get_task_from_mapping(task.name, tasks_mapping)
+            required = td.get("required_env_vars", []) or []
+            for var in required:
+                if var == "API_KEY":
+                    if not (("API_KEY" in lepton_env_vars) or bool(api_key_name)):
+                        raise ValueError(
+                            f"Task '{task.name}' requires API_KEY: set execution.lepton_platform.tasks.env_vars.API_KEY "
+                            "or target.api_endpoint.api_key_name"
+                        )
+                else:
+                    if var not in lepton_env_vars:
+                        raise ValueError(
+                            f"Task '{task.name}' requires {var}: set it under execution.lepton_platform.tasks.env_vars"
+                        )
+
+        # success (use realized output directory if invocation_id is available)
+        preview_output_dir = (
+            Path(cfg.execution.output_dir).absolute() / invocation_id
+            if invocation_id
+            else Path(cfg.execution.output_dir).absolute() / "<invocation_id>"
+        )
+        print(f"   - Tasks: {len(cfg.evaluation.tasks)}")
+        for idx, task in enumerate(cfg.evaluation.tasks):
+            print(f"   - Task {idx}: {task.name}")
+        print(f"   - Output directory: {preview_output_dir}")
+        print("\nTo run evaluation, execute run command without --dry-run")
+    except Exception as e:
+        print(f"❌ Configuration invalid: {e}")
+        logger.error("Lepton dry-run validation failed", error=str(e))
+        return
+
+
 def _get_statuses_for_invocation_id(id: str, db: ExecutionDB) -> List[ExecutionStatus]:
     """Helper method that returns statuses if id is the invocation id"""
     jobs = db.get_jobs(id)

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 15
+PATCH = 17
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.15
+Version: 0.1.17
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.15 → nemo_evaluator_launcher-0.1.17}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ src/nemo_evaluator_launcher/api/functional.py
 src/nemo_evaluator_launcher/api/types.py
 src/nemo_evaluator_launcher/api/utils.py
 src/nemo_evaluator_launcher/cli/__init__.py
+src/nemo_evaluator_launcher/cli/debug.py
 src/nemo_evaluator_launcher/cli/export.py
 src/nemo_evaluator_launcher/cli/kill.py
 src/nemo_evaluator_launcher/cli/ls_runs.py