nemo-evaluator-launcher 0.1.41__py3-none-any.whl → 0.1.67__py3-none-any.whl

nemo_evaluator_launcher/common/container_metadata/utils.py

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Utility functions for container metadata processing."""
+
+
+def parse_container_image(container_image: str) -> tuple[str, str, str, str]:
+    """Parse a container image string into registry type, registry URL, repository, and tag.
+
+    Args:
+        container_image: Container image string (e.g., "nvcr.io/nvidia/eval-factory/simple-evals:25.10")
+
+    Returns:
+        Tuple of (registry_type, registry_url, repository, tag)
+    """
+    # Split tag from image
+    if ":" in container_image:
+        image_part, tag = container_image.rsplit(":", 1)
+    else:
+        image_part = container_image
+        tag = "latest"
+
+    # Parse registry and repository
+    parts = image_part.split("/")
+    if len(parts) < 2:
+        raise ValueError(f"Invalid container image format: {container_image}")
+
+    # Check if first part is a registry (contains '.' or is 'localhost')
+    if "." in parts[0] or parts[0] == "localhost":
+        registry_host = parts[0]
+        # Determine registry type
+        if "gitlab" in registry_host.lower():
+            registry_type = "gitlab"
+        elif "nvcr.io" in registry_host:
+            registry_type = "nvcr"
+        else:
+            registry_type = "nvcr"  # Default to nvcr for other registries
+
+        # Check if registry has a port
+        if ":" in registry_host:
+            registry_url = registry_host
+        else:
+            registry_url = registry_host
+        repository = "/".join(parts[1:])
+    else:
+        # Default registry (Docker Hub)
+        registry_type = "nvcr"
+        registry_url = "registry-1.docker.io"
+        repository = image_part
+
+    return registry_type, registry_url, repository, tag

nemo_evaluator_launcher/common/helpers.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 #
 import base64
-import copy
 import datetime
 from dataclasses import dataclass
 from typing import Optional
@@ -71,6 +70,47 @@ def _set_nested_optionally_overriding(
         temp[keys[-1]] = val
 
 
+_MIGRATION_MESSAGE = """
+`overrides` field is no longer supported. Use `nemo_evaluator_config` field instead, e.g.:
+
+1. If you are using overrides in your yaml config, replace:
+
+```yaml
+evaluation:
+  overrides:
+    config.params.temperature: 0.6
+    config.params.top_p: 0.95
+```
+
+with:
+
+```yaml
+evaluation:
+  nemo_evaluator_config:
+    config:
+      params:
+        temperature: 0.6
+        top_p: 0.95
+```
+
+2. If you are using overrides in your cli command, replace:
+
+```bash
+nemo-evaluator-launcher run --config my_config.yaml \\
+    -o evaluation.overrides.config.params.temperature=0.6 \\
+    -o evaluation.overrides.config.params.top_p=0.95
+```
+
+with:
+
+```bash
+nemo-evaluator-launcher run --config my_config.yaml \\
+    -o evaluation.nemo_evaluator_config.config.params.temperature=0.6 \\
+    -o evaluation.nemo_evaluator_config.config.params.top_p=0.95
+```
+"""
+
+
 def get_eval_factory_config(
     cfg: DictConfig,
     user_task_config: DictConfig,
@@ -79,17 +119,11 @@ def get_eval_factory_config(
 
     This function extracts the config field similar to how overrides are handled.
 
-    Overrides will start to be deprecated (or not, but at least a warning will be logged).
+    It applies task-level overrides to the global overrides.
     """
 
     if cfg.evaluation.get("overrides") or user_task_config.get("overrides"):
-        # TODO(agronskiy): start removing overrides, test `test_start_deprecating_overrides`
-        # will start failing soon.
-        logger.warning(
-            "We are deprecating using old-style dot-delimited overrides "
-            "in favour of `nemo_evaluator_config` field. Please check "
-            "the documentation."
-        )
+        raise ValueError(_MIGRATION_MESSAGE)
 
     logger.debug("Getting nemo evaluator merged config")
     # Extract config fields similar to overrides - convert to basic Python types first
@@ -163,6 +197,7 @@ def get_eval_factory_command(
         ["config", "output_dir"],
         "/results",
     )
+    # FIXME(martas): update to api_key_name after 25.12 is released
     _set_nested_optionally_overriding(
         merged_nemo_evaluator_config,
         ["target", "api_endpoint", "api_key"],
@@ -216,18 +251,6 @@ def get_eval_factory_command(
         + "&& $cmd run_eval --run_config config_ef.yaml"
     )
 
-    # NOTE: see note and test about deprecating that.
-    overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
-    overrides.update(dict(user_task_config.get("overrides", {})))
-    # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
-    # Consider passing a JSON object on Eval Factory side
-    overrides = {
-        k: (v.strip("\n") if isinstance(v, str) else v) for k, v in overrides.items()
-    }
-    overrides_str = ",".join([f"{k}={v}" for k, v in overrides.items()])
-    if overrides_str:
-        eval_command = f"{eval_command} --overrides {overrides_str}"
-
     # We return both the command and the debugging base64-decoded strings, useful
     # for exposing when building scripts.
     return CmdAndReadableComment(
@@ -257,13 +280,6 @@ def get_endpoint_url(
         if nemo_evaluator_config_url:
             return nemo_evaluator_config_url
 
-        # Being deprecated, see `get_eval_factory_config` message.
-        overrides_old_style_url = merged_nemo_evaluator_config.get("overrides", {}).get(
-            "target.api_endpoint.url", None
-        )
-        if overrides_old_style_url:
-            return overrides_old_style_url
-
         return url
 
     if cfg.deployment.type == "none":

nemo_evaluator_launcher/common/mapping.py

@@ -13,222 +13,163 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import importlib
 import pathlib
-import sys
-from importlib import resources
-from typing import Any, Optional
-
-import requests
-
-if sys.version_info >= (3, 11):
-    import tomllib
-else:
-    import tomli as tomllib
+from typing import Any
 
+from nemo_evaluator_launcher.common.container_metadata import (
+    TaskIntermediateRepresentation,
+    load_tasks_from_tasks_file,
+)
 from nemo_evaluator_launcher.common.logging_utils import logger
 
-# Configuration constants
-# For below, see docs: https://docs.github.com/en/rest/repos/contents
-MAPPING_URL = "https://raw.githubusercontent.com/NVIDIA-NeMo/Eval/main/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/resources/mapping.toml"
-CACHE_DIR = pathlib.Path.home() / ".nemo-evaluator" / "cache"
-CACHE_FILENAME = "mapping.toml"
-INTERNAL_RESOURCES_PKG = "nemo_evaluator_launcher_internal.resources"
-
-
-def _ensure_cache_dir() -> None:
-    """Ensure the cache directory exists."""
-    CACHE_DIR.mkdir(parents=True, exist_ok=True)
-
-
-def _get_cache_file() -> pathlib.Path:
-    """Get the cache file path.
-
-    Returns:
-        pathlib.Path: Path to the cache file.
-    """
-    return CACHE_DIR / CACHE_FILENAME
-
-
-def _download_latest_mapping() -> Optional[bytes]:
-    """Download latest mapping from MAPPING_URL and return raw bytes.
-
-    Returns:
-        Optional[bytes]: Downloaded mapping bytes, or None if download fails.
-    """
-    try:
-        response = requests.get(MAPPING_URL, timeout=10)
-        response.raise_for_status()
-
-        # For GitHub raw URLs, the response content is the file content directly
-        mapping_bytes = response.content
-        assert isinstance(mapping_bytes, bytes)
-
-        logger.debug("Successfully downloaded mapping from remote URL")
-        return mapping_bytes
-    except (requests.RequestException, OSError) as e:
-        logger.warning("Failed to download mapping from remote URL", error=str(e))
-        return None
 
+def _convert_irs_to_mapping_format(
+    tasks: list[TaskIntermediateRepresentation],
+) -> dict[tuple[str, str], dict]:
+    """Convert list of TaskIntermediateRepresentation objects to mapping dict format.
 
-def _load_cached_mapping() -> Optional[dict[Any, Any]]:
-    """Load mapping from cache file.
+    Args:
+        tasks: List of TaskIntermediateRepresentation objects.
+        harnesses_by_name: Optional mapping of harness name -> Harness IR. If provided,
+            adds harness-level metadata (e.g., arch) to each task mapping entry.
 
     Returns:
-        Optional[dict]: Loaded mapping data, or None if loading fails.
+        dict: Mapping of (harness_name, task_name) to dict holding their configuration.
     """
-    cache_file = _get_cache_file()
-    if not cache_file.exists():
-        return None
-
-    try:
-        with open(cache_file, "rb") as f:
-            mapping = tomllib.load(f)
-        logger.debug("Loaded mapping from cache")
-        return mapping  # type: ignore[no-any-return]
-    except (OSError, tomllib.TOMLDecodeError) as e:
-        logger.warning("Failed to load mapping from cache", error=str(e))
-        return None
-
+    mapping: dict[tuple[str, str], dict] = {}
+
+    for task_ir in tasks:
+        harness_name = task_ir.harness
+        task_name = task_ir.name
+        key = (harness_name, task_name)
+
+        if key in mapping:
+            logger.warning(
+                "Duplicate task key found in IRs, keeping first occurrence",
+                harness=harness_name,
+                task=task_name,
+            )
+            continue
+
+        # Extract endpoint_type from defaults.config.supported_endpoint_types
+        defaults = task_ir.defaults or {}
+        config = defaults.get("config", {})
+        supported_endpoint_types = config.get("supported_endpoint_types", ["chat"])
+        endpoint_type = (
+            supported_endpoint_types[0] if supported_endpoint_types else "chat"
+        )
 
-def _save_mapping_to_cache(mapping_bytes: bytes) -> None:
-    """Save mapping to cache file.
+        # Extract type from defaults.config.type
+        task_type = config.get("type", "")
 
-    Args:
-        mapping_bytes: Mapping data to save.
-    """
-    try:
-        _ensure_cache_dir()
-        cache_file = _get_cache_file()
+        # Build mapping entry
+        mapping[key] = {
+            "task": task_name,
+            "harness": harness_name,
+            "endpoint_type": endpoint_type,
+            "container": task_ir.container,
+        }
 
-        # Save the mapping data
-        with open(cache_file, "wb") as f:
-            f.write(mapping_bytes)
+        if task_ir.container_arch:
+            mapping[key]["arch"] = task_ir.container_arch
 
-    except OSError as e:
-        logger.warning("Failed to save mapping to cache", error=str(e))
+        # Backwards-compatible enhancement: keep full IR defaults available.
+        # Existing code uses flattened defaults (excluding `config`) below; this adds a
+        # new field without changing any existing keys.
+        mapping[key]["defaults"] = defaults
 
+        # Backwards-compatible enhancement: surface command explicitly if present.
+        # Note: `command` is already included via flattened defaults merge, but
+        # keeping it explicit makes downstream usage simpler.
+        if "command" in defaults and "command" not in mapping[key]:
+            mapping[key]["command"] = defaults["command"]
 
-def _load_packaged_resource(
-    resource_name: str, pkg_name: str = "nemo_evaluator_launcher.resources"
-) -> dict[str, Any]:
-    """Load a resource from the packaged resources.
+        # Add description if available
+        if task_ir.description:
+            mapping[key]["description"] = task_ir.description
 
-    Args:
-        resource_name: The name of the resource to load.
-    """
-    try:
-        resource_toml: dict[str, Any] = {}
-        with resources.files(pkg_name).joinpath(resource_name).open("rb") as f:
-            resource_toml = tomllib.load(f)
-        logger.info(
-            "Loaded resource from packaged file", resource=resource_name, pkg=pkg_name
-        )
-        return resource_toml
-    except (OSError, tomllib.TOMLDecodeError) as e:
-        logger.error(
-            "Failed to load from packaged file",
-            resource=resource_name,
-            pkg=pkg_name,
-            error=str(e),
-        )
-        raise RuntimeError(f"Failed to load {resource_name} from packaged file") from e
+        # Add type if available
+        if task_type:
+            mapping[key]["type"] = task_type
 
+        # Add container_digest if available
+        if task_ir.container_digest:
+            mapping[key]["container_digest"] = task_ir.container_digest
 
-def _process_mapping(mapping_toml: dict) -> dict:
-    """Process the raw mapping TOML into the expected format.
+        # Merge defaults (flattened, excluding config which is already processed)
+        defaults_copy = {k: v for k, v in defaults.items() if k != "config"}
+        mapping[key].update(defaults_copy)
 
-    Args:
-        mapping_toml: Raw mapping TOML data.
-    Returns:
-        dict: Processed mapping in the expected format.
-    """
-    mapping = {}
-    for harness_name, harness_data in mapping_toml.items():
-        assert isinstance(harness_data["tasks"], dict)
-        for endpoint_type, harness_tasks in harness_data["tasks"].items():
-            assert isinstance(harness_tasks, dict)
-            for task_name, task_data in harness_tasks.items():
-                assert isinstance(task_data, dict)
-                key = (harness_name, task_name)
-                if key in mapping:
-                    raise KeyError(
-                        f"(harness,task)-tuple key {repr(key)} already exists in the mapping"
-                    )
-                mapping[key] = {
-                    "task": task_name,
-                    "harness": harness_name,
-                    "container": harness_data["container"],
-                    "endpoint_type": endpoint_type,
-                }
-                for task_data_key in task_data.keys():
-                    if task_data_key in mapping[key]:
-                        raise KeyError(
-                            f"{repr(task_data_key)} is not allowed as key under {repr(key)} in the mapping"
-                        )
-                mapping[key].update(task_data)
     return mapping
 
 
 def load_tasks_mapping(
-    latest: bool = False,
     mapping_toml: pathlib.Path | str | None = None,
+    *,
+    from_container: str | None = None,
 ) -> dict[tuple[str, str], dict]:
     """Load tasks mapping.
 
     The function obeys the following priority rules:
-    1. (Default) If latest==False and mapping_toml is None -> load packaged mapping.
-    2. If latest==True -> fetch MAPPING_URL, save to cache, load it.
-    3. If mapping_toml is not None -> load mapping from this path.
+    1. If from_container is not None -> extract framework.yml from that container and build mapping from the resulting IRs.
+    2. Otherwise -> load packaged IRs (all_tasks_irs.yaml) and build mapping from those IRs.
+
+    Args:
+        mapping_toml: Deprecated. mapping.toml is no longer supported (IR-only mode).
+        from_container: Optional container image identifier. If provided, tasks are loaded on-the-fly from that container.
 
     Returns:
         dict: Mapping of (harness_name, task_name) to dict holding their configuration.
 
     """
-    local_mapping: dict = {}
-    if latest:
-        mapping_bytes = _download_latest_mapping()
-        if mapping_bytes:
-            _save_mapping_to_cache(mapping_bytes)
-            local_mapping = _process_mapping(
-                tomllib.loads(mapping_bytes.decode("utf-8"))
+    if mapping_toml is not None:
+        raise ValueError(
+            "mapping_toml is no longer supported. This project has switched to packaged IRs (all_tasks_irs.yaml)."
+        )
+
+    # Explicit container path: extract tasks from container and return mapping built from IRs.
+    # This bypasses packaged IRs.
+    if from_container is not None:
+        try:
+            # Optional dependency path; importing may fail in "IR-only" environments.
+            from nemo_evaluator_launcher.common.container_metadata import (
+                load_tasks_from_container,
+            )
+        except ModuleNotFoundError as e:
+            raise RuntimeError(
+                "Loading tasks from a container requires optional dependencies. "
+                "Install nemo-evaluator-launcher with the full runtime dependencies."
+            ) from e
+
+        tasks = load_tasks_from_container(from_container)
+        if not tasks:
+            logger.warning(
+                "No tasks loaded from container via container-metadata",
+                container=from_container,
             )
         else:
-            # Fallback to cached mapping; raise only if cache is missing/invalid
-            cached = _load_cached_mapping()
-            if cached:
-                local_mapping = _process_mapping(cached)
-            else:
-                raise RuntimeError("could not download latest mapping")
-
-    elif mapping_toml is not None:
-        with open(mapping_toml, "rb") as f:
-            local_mapping = _process_mapping(tomllib.load(f))
-    else:
-        local_mapping = _process_mapping(_load_packaged_resource(CACHE_FILENAME))
+            logger.info(
+                "Loaded tasks from container via container-metadata",
+                container=from_container,
+                num_tasks=len(tasks),
+            )
 
-    # TODO: make more elegant. We consider it ok to avoid a fully-blown plugin system.
-    # Check if nemo_evaluator_launcher_internal is available and load its mapping.toml
-    # CAVEAT: lazy-loading here, not somewhere top level, is important, to ensure
-    # order of package initialization.
-    try:
-        importlib.import_module("nemo_evaluator_launcher_internal")
-        logger.debug("Internal package available, loading internal mapping")
-        internal_mapping = _process_mapping(
-            _load_packaged_resource(CACHE_FILENAME, INTERNAL_RESOURCES_PKG)
-        )
+        return _convert_irs_to_mapping_format(tasks)
 
-        # Merge internal mapping with local mapping (internal takes precedence)
-        local_mapping.update(internal_mapping)
-        logger.info(
-            "Successfully merged internal mapping", internal_tasks=len(internal_mapping)
-        )
-    except ImportError:
-        logger.debug("Internal package not available, using external mapping only")
+    try:
+        tasks, mapping_verified = load_tasks_from_tasks_file()
     except Exception as e:
-        logger.warning("Failed to load internal mapping", error=str(e))
+        raise RuntimeError("Failed to load tasks from packaged IRs") from e
+
+    if not tasks:
+        raise RuntimeError("No tasks available in packaged IRs (all_tasks_irs.yaml)")
 
-    return local_mapping
+    logger.info(
+        "Loaded tasks from packaged IRs",
+        num_tasks=len(tasks),
+        mapping_verified=mapping_verified,
+    )
+    return _convert_irs_to_mapping_format(tasks)
 
 
 def get_task_from_mapping(query: str, mapping: dict[Any, Any]) -> dict[Any, Any]:
@@ -293,3 +234,51 @@ def get_task_from_mapping(query: str, mapping: dict[Any, Any]) -> dict[Any, Any]
             f"invalid query={repr(query)} for task mapping,"
             " it must contain exactly zero or one occurrence of '.' character"
         )
+
+
+def _minimal_task_definition(task_query: str, *, container: str) -> dict[str, Any]:
+    """Create a minimal task definition when task is not known in any mapping."""
+    if task_query.count(".") == 1:
+        harness, task = task_query.split(".")
+    else:
+        harness, task = "", task_query
+
+    # Default to chat; most configs and endpoints use chat unless explicitly known.
+    return {
+        "task": task,
+        "harness": harness,
+        "endpoint_type": "chat",
+        "container": container,
+    }
+
+
+def get_task_definition_for_job(
+    *,
+    task_query: str,
+    base_mapping: dict[Any, Any],
+    container: str | None = None,
+) -> dict[str, Any]:
+    """Resolve task definition for a job.
+
+    If a container is provided, tasks are loaded from that container (using
+    container-metadata) and we attempt to resolve the task from that mapping.
+    If the task isn't found in the container, we warn and return a minimal
+    task definition so submission can proceed.
+    """
+    if not container:
+        return get_task_from_mapping(task_query, base_mapping)
+
+    # `load_tasks_mapping(from_container=...)` uses container-metadata extraction,
+    # which already has its own caching (e.g., caching extracted framework.yml).
+    container_mapping = load_tasks_mapping(from_container=container)
+
+    try:
+        return get_task_from_mapping(task_query, container_mapping)
+    except ValueError as e:
+        logger.warning(
+            "Task not found in provided container; proceeding with minimal task definition",
+            task=task_query,
+            container=container,
+            error=str(e),
+        )
+        return _minimal_task_definition(task_query, container=container)

nemo_evaluator_launcher/common/printing_utils.py

@@ -62,39 +62,45 @@ _CODES: dict[str, str] = dict(
     reset="\033[0m",
 )
 
-# If the colors are disabled, we null-out all the codes.
-if _is_color_disabled():
-    for c in _CODES.keys():
-        _CODES[c] = ""
+
+def _apply(code_key: str, s: str) -> str:
+    """Apply an ANSI code if colors are enabled.
+
+    Note: Color enablement is evaluated at call time so tests that redirect
+    stdout (e.g. to StringIO) correctly disable ANSI sequences.
+    """
+    if _is_color_disabled():
+        return s
+    return _CODES[code_key] + s + _CODES["reset"]
 
 
 def green(s: str) -> str:
-    return _CODES["green"] + s + _CODES["reset"]
+    return _apply("green", s)
 
 
 def red(s: str) -> str:
-    return _CODES["red"] + s + _CODES["reset"]
+    return _apply("red", s)
 
 
 def red_bg(s: str) -> str:
-    return _CODES["red_bg"] + s + _CODES["reset"]
+    return _apply("red_bg", s)
 
 
 def cyan(s: str) -> str:
-    return _CODES["cyan"] + s + _CODES["reset"]
+    return _apply("cyan", s)
 
 
 def yellow(s: str) -> str:
-    return _CODES["yellow"] + s + _CODES["reset"]
+    return _apply("yellow", s)
 
 
 def magenta(s: str) -> str:
-    return _CODES["magenta"] + s + _CODES["reset"]
+    return _apply("magenta", s)
 
 
 def grey(s: str) -> str:
-    return _CODES["grey"] + s + _CODES["reset"]
+    return _apply("grey", s)
 
 
 def bold(s: str) -> str:
-    return _CODES["bold"] + s + _CODES["reset"]
+    return _apply("bold", s)

nemo_evaluator_launcher/configs/deployment/nim.yaml

@@ -18,13 +18,15 @@ image: ??? # e.g., nvcr.io/nim/meta/llama-3.1-8b-instruct:1.8.6
 served_model_name: ???
 port: 8000
 
+command: /opt/nim/start_server.sh
+
 # NIM containers use default entrypoint - no custom command needed
 # Configuration is done via environment variables in lepton_config
 
 endpoints:
   chat: /v1/chat/completions
   completions: /v1/completions
-  health: /health
+  health: /v1/health/ready
 # Note: Environment variables should be configured in lepton_config.envs
 # Auto-derived environment variables from deployment config:
 # - SERVED_MODEL_NAME (from served_model_name)