nemo-evaluator-launcher 0.1.41__py3-none-any.whl → 0.1.56__py3-none-any.whl

nemo_evaluator_launcher/common/container_metadata/utils.py

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Utility functions for container metadata processing."""
+
+
+def parse_container_image(container_image: str) -> tuple[str, str, str, str]:
+    """Parse a container image string into registry type, registry URL, repository, and tag.
+
+    Args:
+        container_image: Container image string (e.g., "nvcr.io/nvidia/eval-factory/simple-evals:25.10")
+
+    Returns:
+        Tuple of (registry_type, registry_url, repository, tag)
+    """
+    # Split tag from image
+    if ":" in container_image:
+        image_part, tag = container_image.rsplit(":", 1)
+    else:
+        image_part = container_image
+        tag = "latest"
+
+    # Parse registry and repository
+    parts = image_part.split("/")
+    if len(parts) < 2:
+        raise ValueError(f"Invalid container image format: {container_image}")
+
+    # Check if first part is a registry (contains '.' or is 'localhost')
+    if "." in parts[0] or parts[0] == "localhost":
+        registry_host = parts[0]
+        # Determine registry type
+        if "gitlab" in registry_host.lower():
+            registry_type = "gitlab"
+        elif "nvcr.io" in registry_host:
+            registry_type = "nvcr"
+        else:
+            registry_type = "nvcr"  # Default to nvcr for other registries
+
+        # Check if registry has a port
+        if ":" in registry_host:
+            registry_url = registry_host
+        else:
+            registry_url = registry_host
+        repository = "/".join(parts[1:])
+    else:
+        # Default registry (Docker Hub)
+        registry_type = "nvcr"
+        registry_url = "registry-1.docker.io"
+        repository = image_part
+
+    return registry_type, registry_url, repository, tag

nemo_evaluator_launcher/common/helpers.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 #
 import base64
-import copy
 import datetime
 from dataclasses import dataclass
 from typing import Optional
@@ -71,6 +70,47 @@ def _set_nested_optionally_overriding(
         temp[keys[-1]] = val
 
 
+_MIGRATION_MESSAGE = """
+`overrides` field is no longer supported. Use `nemo_evaluator_config` field instead, e.g.:
+
+1. If you are using overrides in your yaml config, replace:
+
+```yaml
+evaluation:
+  overrides:
+    config.params.temperature: 0.6
+    config.params.top_p: 0.95
+```
+
+with:
+
+```yaml
+evaluation:
+  nemo_evaluator_config:
+    config:
+      params:
+        temperature: 0.6
+        top_p: 0.95
+```
+
+2. If you are using overrides in your cli command, replace:
+
+```bash
+nemo-evaluator-launcher run --config my_config.yaml \\
+    -o evaluation.overrides.config.params.temperature=0.6 \\
+    -o evaluation.overrides.config.params.top_p=0.95
+```
+
+with:
+
+```bash
+nemo-evaluator-launcher run --config my_config.yaml \\
+    -o evaluation.nemo_evaluator_config.config.params.temperature=0.6 \\
+    -o evaluation.nemo_evaluator_config.config.params.top_p=0.95
+```
+"""
+
+
 def get_eval_factory_config(
     cfg: DictConfig,
     user_task_config: DictConfig,
@@ -79,17 +119,11 @@ def get_eval_factory_config(
 
     This function extracts the config field similar to how overrides are handled.
 
-    Overrides will start to be deprecated (or not, but at least a warning will be logged).
+    It applies task-level overrides to the global overrides.
     """
 
     if cfg.evaluation.get("overrides") or user_task_config.get("overrides"):
-        # TODO(agronskiy): start removing overrides, test `test_start_deprecating_overrides`
-        # will start failing soon.
-        logger.warning(
-            "We are deprecating using old-style dot-delimited overrides "
-            "in favour of `nemo_evaluator_config` field. Please check "
-            "the documentation."
-        )
+        raise ValueError(_MIGRATION_MESSAGE)
 
     logger.debug("Getting nemo evaluator merged config")
     # Extract config fields similar to overrides - convert to basic Python types first
@@ -163,6 +197,7 @@ def get_eval_factory_command(
         ["config", "output_dir"],
         "/results",
     )
+    # FIXME(martas): update to api_key_name after 25.12 is released
     _set_nested_optionally_overriding(
         merged_nemo_evaluator_config,
         ["target", "api_endpoint", "api_key"],
@@ -216,18 +251,6 @@ def get_eval_factory_command(
         + "&& $cmd run_eval --run_config config_ef.yaml"
     )
 
-    # NOTE: see note and test about deprecating that.
-    overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
-    overrides.update(dict(user_task_config.get("overrides", {})))
-    # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
-    # Consider passing a JSON object on Eval Factory side
-    overrides = {
-        k: (v.strip("\n") if isinstance(v, str) else v) for k, v in overrides.items()
-    }
-    overrides_str = ",".join([f"{k}={v}" for k, v in overrides.items()])
-    if overrides_str:
-        eval_command = f"{eval_command} --overrides {overrides_str}"
-
     # We return both the command and the debugging base64-decoded strings, useful
     # for exposing when building scripts.
     return CmdAndReadableComment(
@@ -257,13 +280,6 @@ def get_endpoint_url(
         if nemo_evaluator_config_url:
             return nemo_evaluator_config_url
 
-        # Being deprecated, see `get_eval_factory_config` message.
-        overrides_old_style_url = merged_nemo_evaluator_config.get("overrides", {}).get(
-            "target.api_endpoint.url", None
-        )
-        if overrides_old_style_url:
-            return overrides_old_style_url
-
         return url
 
     if cfg.deployment.type == "none":