nemo-evaluator-launcher 0.1.0rc3__tar.gz → 0.1.0rc5__tar.gz

{nemo_evaluator_launcher-0.1.0rc3/src/nemo_evaluator_launcher.egg-info → nemo_evaluator_launcher-0.1.0rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc3
+Version: 0.1.0rc5
 Summary: NeMo Evaluator Launcher - Public API
 Requires-Python: <3.14,>=3.10
 Description-Content-Type: text/markdown
@@ -32,4 +32,4 @@ Dynamic: license-file
 
 # NeMo Evaluator Launcher
 
-For complete documentation, please see: [docs/nemo-evaluator-launcher/index.md](../../docs/nemo-evaluator-launcher/index.md)
+For complete documentation, please see: [docs/nemo-evaluator-launcher/index.md](https://github.com/NVIDIA-NeMo/Eval/tree/main/docs/nemo-evaluator-launcher/index.md)

nemo_evaluator_launcher-0.1.0rc5/README.md

@@ -0,0 +1,3 @@
+# NeMo Evaluator Launcher
+
+For complete documentation, please see: [docs/nemo-evaluator-launcher/index.md](https://github.com/NVIDIA-NeMo/Eval/tree/main/docs/nemo-evaluator-launcher/index.md)

{nemo_evaluator_launcher-0.1.0rc3 → nemo_evaluator_launcher-0.1.0rc5}/src/nemo_evaluator_launcher/common/helpers.py

@@ -13,18 +13,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import base64
 import copy
 import datetime
 from typing import Optional
 
+import yaml
 from omegaconf import DictConfig, OmegaConf
 
 from nemo_evaluator_launcher.common.logging_utils import logger
 
 
+def _yaml_to_echo_command(yaml_str: str, filename: str = "config_ef.yaml") -> str:
+    yaml_str_b64 = base64.b64encode(yaml_str.encode("utf-8")).decode("utf-8")
+    return f'echo "{yaml_str_b64}" | base64 -d > {filename}'
+
+
+def get_eval_factory_config(
+    cfg: DictConfig, user_task_config: DictConfig, task_definition: dict
+) -> dict:
+    """Extract config fields for eval factory.
+
+    This function extracts the config field similar to how overrides are handled.
+    """
+    # Extract config fields similar to overrides - convert to basic Python types first
+    cfg_config = cfg.evaluation.get("config", {})
+    user_config = user_task_config.get("config", {})
+
+    # Convert OmegaConf objects to basic Python types
+    if cfg_config:
+        cfg_config = OmegaConf.to_container(cfg_config, resolve=True)
+    if user_config:
+        user_config = OmegaConf.to_container(user_config, resolve=True)
+
+    # Merge the configs
+    config_fields = copy.deepcopy(cfg_config or {})
+    config_fields.update(user_config or {})
+
+    return config_fields
+
+
 def get_eval_factory_command(
     cfg: DictConfig, user_task_config: DictConfig, task_definition: dict
 ) -> str:
+    config_fields = get_eval_factory_config(cfg, user_task_config, task_definition)
+
     overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
     overrides.update(dict(user_task_config.get("overrides", {})))
     # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
@@ -34,16 +67,20 @@ def get_eval_factory_command(
     }
     overrides_str = ",".join([f"{k}={v}" for k, v in overrides.items()])
     model_url = get_endpoint_url(cfg, user_task_config, task_definition)
-    command = f"""nv_eval run_eval \
-    --model_id {get_served_model_name(cfg)} \
-    --model_type {task_definition["endpoint_type"]} \
-    --eval_type {task_definition["task"]} \
-    --model_url {model_url} \
-    --api_key_name API_KEY \
-    --output_dir /results"""
-    if overrides_str:
-        command = f"{command} --overrides {overrides_str}"
-    return command
+
+    model_id = get_served_model_name(cfg)
+    model_type = task_definition["endpoint_type"]
+    eval_type = task_definition["task"]
+
+    create_file_cmd = _yaml_to_echo_command(
+        yaml.safe_dump(config_fields), "config_ef.yaml"
+    )
+    nv_eval_command = f"""nv_eval run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
+
+    if overrides:
+        nv_eval_command = f"{nv_eval_command} --overrides {overrides_str}"
+
+    return create_file_cmd + " && " + "cat config_ef.yaml && " + nv_eval_command
 
 
 def get_endpoint_url(

{nemo_evaluator_launcher-0.1.0rc3 → nemo_evaluator_launcher-0.1.0rc5}/src/nemo_evaluator_launcher/executors/slurm/executor.py

@@ -594,8 +594,9 @@ def _create_slurm_sbatch_script(
         s += "--no-container-mount-home "
     s += "--container-mounts {} ".format(",".join(evaluation_mounts_list))
     s += "--output {} ".format(remote_task_subdir / "logs" / "client-%A.out")
+    s += "bash -c '"
     s += get_eval_factory_command(cfg, task, task_definition)
-    s += "\n\n"
+    s += "'\n\n"
 
     # terminate the server after all evaluation clients finish
     if cfg.deployment.type != "none":

{nemo_evaluator_launcher-0.1.0rc3 → nemo_evaluator_launcher-0.1.0rc5}/src/nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 MAJOR = 0
 MINOR = 1
 PATCH = 0
-PRE_RELEASE = "rc3"
+PRE_RELEASE = "rc5"
 
 # Use the following formatting: (major, minor, patch, pre-release)
 VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE)

{nemo_evaluator_launcher-0.1.0rc3 → nemo_evaluator_launcher-0.1.0rc5/src/nemo_evaluator_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc3
+Version: 0.1.0rc5
 Summary: NeMo Evaluator Launcher - Public API
 Requires-Python: <3.14,>=3.10
 Description-Content-Type: text/markdown
@@ -32,4 +32,4 @@ Dynamic: license-file
 
 # NeMo Evaluator Launcher
 
-For complete documentation, please see: [docs/nemo-evaluator-launcher/index.md](../../docs/nemo-evaluator-launcher/index.md)
+For complete documentation, please see: [docs/nemo-evaluator-launcher/index.md](https://github.com/NVIDIA-NeMo/Eval/tree/main/docs/nemo-evaluator-launcher/index.md)

nemo_evaluator_launcher-0.1.0rc3/README.md

@@ -1,3 +0,0 @@
-# NeMo Evaluator Launcher
-
-For complete documentation, please see: [docs/nemo-evaluator-launcher/index.md](../../docs/nemo-evaluator-launcher/index.md)