nemo-evaluator 0.1.71__py3-none-any.whl → 0.1.73__py3-none-any.whl

nemo_evaluator/core/entrypoint.py

@@ -30,6 +30,7 @@ from nemo_evaluator.api.api_dataclasses import (
 from nemo_evaluator.core.evaluate import evaluate
 from nemo_evaluator.core.input import (
     _get_framework_evaluations,
+    _is_internal_package_installed,
     _parse_cli_args,
     load_run_config,
     validate_configuration,
@@ -136,11 +137,16 @@ def show_available_tasks() -> None:
     if not core_evals_pkg:
         print("NO evaluation packages are installed.")
 
+    include_internal = _is_internal_package_installed()
     for pkg in core_evals_pkg:
         framework_eval_mapping, *_ = _get_framework_evaluations(
-            os.path.join(pkg.module_finder.path, pkg.name, "framework.yml")
+            os.path.join(pkg.module_finder.path, pkg.name, "framework.yml"),
+            include_internal=include_internal,
         )
         for ind_pkg in framework_eval_mapping.keys():
+            # Empty task mapping means harness has no public tasks.
+            if not framework_eval_mapping[ind_pkg]:
+                continue
             print(f"{ind_pkg}: ")
             for task in framework_eval_mapping[ind_pkg].keys():
                 print(f"  * {task}")
@@ -160,9 +166,6 @@ def run(args) -> None:
         print(cmd)
         exit(0)
 
-    # Validate configuration first (catches config errors early, before evaluate())
-    validate_configuration(run_config)
-
     metadata_cfg: EvaluationMetadata | None = run_config.get("metadata")
 
     # Build evaluation configuration with framework defaults merged

nemo_evaluator/core/input.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import copy
+import importlib.util
 import os
 import pkgutil
 from typing import Optional
@@ -112,7 +113,13 @@ def parse_override_params(override_params_str: Optional[str] = None) -> dict:
     return dotlist_to_dict(pairs)
 
 
-def get_framework_evaluations(filepath: str) -> tuple[str, dict, dict[str, Evaluation]]:
+def _is_internal_package_installed() -> bool:
+    return importlib.util.find_spec("nemo_evaluator_internal") is not None
+
+
+def get_framework_evaluations(
+    filepath: str, *, include_internal: bool = True
+) -> tuple[str, dict, dict[str, Evaluation]]:
     framework = {}
     with open(filepath, "r") as f:
         framework = yaml.safe_load(f)
@@ -124,6 +131,8 @@ def get_framework_evaluations(filepath: str) -> tuple[str, dict, dict[str, Evalu
     run_config_framework_defaults["pkg_name"] = pkg_name
     evaluations = dict()
     for evaluation_dict in framework["evaluations"]:
+        if evaluation_dict.get("internal", False) and not include_internal:
+            continue
         # Apply run config evaluation defaults onto the framework defaults
         run_config_task_defaults = deep_update(
             run_config_framework_defaults, evaluation_dict["defaults"], skip_nones=True
@@ -139,7 +148,7 @@ def get_framework_evaluations(filepath: str) -> tuple[str, dict, dict[str, Evalu
 
 # improve typing
 def _get_framework_evaluations(
-    def_file: str,
+    def_file: str, *, include_internal: bool = True
 ) -> tuple[dict[str, dict[str, Evaluation]], dict[str, dict], dict[str, Evaluation]]:
     # we should decide if this should raise at this point.
     # Probably not because this function is used with task invocation that might
@@ -155,7 +164,7 @@ def _get_framework_evaluations(
         framework_name,
         framework_defaults,
         framework_evaluations,
-    ) = get_framework_evaluations(def_file)
+    ) = get_framework_evaluations(def_file, include_internal=include_internal)
     framework_eval_mapping[framework_name] = framework_evaluations
     eval_name_mapping.update(framework_evaluations)
     framework_defaults = {framework_name: framework_defaults}
@@ -368,7 +377,6 @@ def get_evaluation(
             ]
         except KeyError:
             default_evaluation = Evaluation(**raw_framework_defaults)
-            evaluation_config.type = evaluation_name
             default_evaluation.config.params.task = evaluation_name
     else:
         if isinstance(all_eval_name_mapping[evaluation_name], list):

nemo_evaluator/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 71
+PATCH = 73
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator-0.1.71.dist-info → nemo_evaluator-0.1.73.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator
-Version: 0.1.71
+Version: 0.1.73
 Summary: Common utilities for NVIDIA evaluation frameworks
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator-0.1.71.dist-info → nemo_evaluator-0.1.73.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nemo_evaluator/__init__.py,sha256=lwW_lnmHxRBVEWkT1g71icmaxDR1_DqMrh4wSR0vk58,1787
-nemo_evaluator/package_info.py,sha256=Y99ow8pPsr3RBeoi1ffoE4154GRn4TWv0l9nd_-B9ns,1525
+nemo_evaluator/package_info.py,sha256=EO8IUhWhQtBdjTqY4qU8dFKH5L_PH08uwNK2Ke8Nyq8,1525
 nemo_evaluator/adapters/__init__.py,sha256=Qbiv8oknyr5yqpWp1npFw3p6EDwZnKUwL3VZXQGtjos,685
 nemo_evaluator/adapters/adapter_config.py,sha256=kyL2KVi7Zc6sn8BomujTY54SK-0RoCZk5v4gS7KHbYk,28007
 nemo_evaluator/adapters/decorators.py,sha256=DeeDW4YscGHNbjq-GyW60WXSfu6TKLaWLCyrmWkqpvw,5648
@@ -31,9 +31,9 @@ nemo_evaluator/client/__init__.py,sha256=IAgXYWT3RwimEjBy80mFC-nLDCUa5KzWVhVX9As
 nemo_evaluator/client/adapter_transport.py,sha256=CybHNbmRxLo2fwDMqbOp8ndYFSKQ_W4ckThmOmmr0Vg,11664
 nemo_evaluator/client/client.py,sha256=DkJjHGzkHWENrLC6Sx8h8HwQRjTAtW_7gpjv4_ZNiO4,12940
 nemo_evaluator/core/__init__.py,sha256=z7nZ70W5kCiFqkpxrOAt2IQfP61II94KIZYf3HwS4w4,120
-nemo_evaluator/core/entrypoint.py,sha256=5uOSqgNQX-LnqMX_eXU4leUvTTITQap0pOi01_D_qbg,7498
+nemo_evaluator/core/entrypoint.py,sha256=4kZCuBOgZBuK9A19k9tTIgGoVZG8fRHp7NCtxVpxjCk,7659
 nemo_evaluator/core/evaluate.py,sha256=49bZ0xIIGl2e9ClE6tLZtVqmQ2yEORiGOLLMpdQtMK8,13124
-nemo_evaluator/core/input.py,sha256=wGIlmlXnDA_fXH3jleQsEPVB-ySUSSnBp8ASscOzfMI,19946
+nemo_evaluator/core/input.py,sha256=1ciyjDShq8OdfzxF5GoKve2LPUWkcFzpDixNNsRBqN8,20243
 nemo_evaluator/core/resources.py,sha256=N8ZBpSUuGFIVSQj7pD0jKeFtxqWPWEhxjpQMVfd1RlY,10441
 nemo_evaluator/core/utils.py,sha256=qnxIsu3-xqgqmAMpqrqpeebu70qqmwCwscRYvZRk2AI,12707
 nemo_evaluator/logging/__init__.py,sha256=s4uGLnSZiSsHJiT42xfwYJAnEfZWVGmgYaEMEkJxVrc,1285
@@ -46,9 +46,9 @@ nemo_evaluator/resources/output_tpl.py,sha256=7gIjcHy2BUTGu5hdf2zLkJgcHck8b4mPvi
 nemo_evaluator/sandbox/__init__.py,sha256=J6fFnQwSgOOsSsudWcRgY8LjFKoiwYRh6UFAiwKmxM0,1066
 nemo_evaluator/sandbox/base.py,sha256=AwAlWcKdSJKLn3DrvENgCUrxcWFu7ItO1NqJBS1z0cY,3199
 nemo_evaluator/sandbox/ecs_fargate.py,sha256=DI6QToYngqrzl8vf1I4GRj0AwDn1YQO64g36-6naZm8,49785
-nemo_evaluator-0.1.71.dist-info/licenses/LICENSE,sha256=COyFnIvgPj0PZK5t89Z_f9mQBR4LH2rDJAkVvR3m7gM,11348
-nemo_evaluator-0.1.71.dist-info/METADATA,sha256=aUWB9yJRvEsfCk0BrJPIC4ufUVopj5nN2VxCSEjCYjI,14203
-nemo_evaluator-0.1.71.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-nemo_evaluator-0.1.71.dist-info/entry_points.txt,sha256=lDcQ6UDEwgoyYdkDx3_YJw8Bzipl_xfVOp3U24IqgcU,289
-nemo_evaluator-0.1.71.dist-info/top_level.txt,sha256=Y-IJy0kY4bd0bay8J_cGaZmxNSjZN3MiFhW7B3VyTeU,15
-nemo_evaluator-0.1.71.dist-info/RECORD,,
+nemo_evaluator-0.1.73.dist-info/licenses/LICENSE,sha256=COyFnIvgPj0PZK5t89Z_f9mQBR4LH2rDJAkVvR3m7gM,11348
+nemo_evaluator-0.1.73.dist-info/METADATA,sha256=_-0b5IGLkhOuIUB04GfznSIezTNQIAA76rvptTifxqg,14203
+nemo_evaluator-0.1.73.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+nemo_evaluator-0.1.73.dist-info/entry_points.txt,sha256=lDcQ6UDEwgoyYdkDx3_YJw8Bzipl_xfVOp3U24IqgcU,289
+nemo_evaluator-0.1.73.dist-info/top_level.txt,sha256=Y-IJy0kY4bd0bay8J_cGaZmxNSjZN3MiFhW7B3VyTeU,15
+nemo_evaluator-0.1.73.dist-info/RECORD,,