nemo-evaluator-launcher 0.1.0rc4__py3-none-any.whl → 0.1.0rc5__py3-none-any.whl

nemo_evaluator_launcher/common/helpers.py

@@ -13,18 +13,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import base64
 import copy
 import datetime
 from typing import Optional
 
+import yaml
 from omegaconf import DictConfig, OmegaConf
 
 from nemo_evaluator_launcher.common.logging_utils import logger
 
 
+def _yaml_to_echo_command(yaml_str: str, filename: str = "config_ef.yaml") -> str:
+    yaml_str_b64 = base64.b64encode(yaml_str.encode("utf-8")).decode("utf-8")
+    return f'echo "{yaml_str_b64}" | base64 -d > {filename}'
+
+
+def get_eval_factory_config(
+    cfg: DictConfig, user_task_config: DictConfig, task_definition: dict
+) -> dict:
+    """Extract config fields for eval factory.
+
+    This function extracts the config field similar to how overrides are handled.
+    """
+    # Extract config fields similar to overrides - convert to basic Python types first
+    cfg_config = cfg.evaluation.get("config", {})
+    user_config = user_task_config.get("config", {})
+
+    # Convert OmegaConf objects to basic Python types
+    if cfg_config:
+        cfg_config = OmegaConf.to_container(cfg_config, resolve=True)
+    if user_config:
+        user_config = OmegaConf.to_container(user_config, resolve=True)
+
+    # Merge the configs
+    config_fields = copy.deepcopy(cfg_config or {})
+    config_fields.update(user_config or {})
+
+    return config_fields
+
+
 def get_eval_factory_command(
     cfg: DictConfig, user_task_config: DictConfig, task_definition: dict
 ) -> str:
+    config_fields = get_eval_factory_config(cfg, user_task_config, task_definition)
+
     overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
     overrides.update(dict(user_task_config.get("overrides", {})))
     # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
@@ -34,16 +67,20 @@ def get_eval_factory_command(
     }
     overrides_str = ",".join([f"{k}={v}" for k, v in overrides.items()])
     model_url = get_endpoint_url(cfg, user_task_config, task_definition)
-    command = f"""nv_eval run_eval \
-    --model_id {get_served_model_name(cfg)} \
-    --model_type {task_definition["endpoint_type"]} \
-    --eval_type {task_definition["task"]} \
-    --model_url {model_url} \
-    --api_key_name API_KEY \
-    --output_dir /results"""
-    if overrides_str:
-        command = f"{command} --overrides {overrides_str}"
-    return command
+
+    model_id = get_served_model_name(cfg)
+    model_type = task_definition["endpoint_type"]
+    eval_type = task_definition["task"]
+
+    create_file_cmd = _yaml_to_echo_command(
+        yaml.safe_dump(config_fields), "config_ef.yaml"
+    )
+    nv_eval_command = f"""nv_eval run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
+
+    if overrides:
+        nv_eval_command = f"{nv_eval_command} --overrides {overrides_str}"
+
+    return create_file_cmd + " && " + "cat config_ef.yaml && " + nv_eval_command
 
 
 def get_endpoint_url(

nemo_evaluator_launcher/executors/slurm/executor.py

@@ -594,8 +594,9 @@ def _create_slurm_sbatch_script(
         s += "--no-container-mount-home "
     s += "--container-mounts {} ".format(",".join(evaluation_mounts_list))
     s += "--output {} ".format(remote_task_subdir / "logs" / "client-%A.out")
+    s += "bash -c '"
     s += get_eval_factory_command(cfg, task, task_definition)
-    s += "\n\n"
+    s += "'\n\n"
 
     # terminate the server after all evaluation clients finish
     if cfg.deployment.type != "none":

nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 MAJOR = 0
 MINOR = 1
 PATCH = 0
-PRE_RELEASE = "rc4"
+PRE_RELEASE = "rc5"
 
 # Use the following formatting: (major, minor, patch, pre-release)
 VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE)

{nemo_evaluator_launcher-0.1.0rc4.dist-info → nemo_evaluator_launcher-0.1.0rc5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.0rc4
+Version: 0.1.0rc5
 Summary: NeMo Evaluator Launcher - Public API
 Requires-Python: <3.14,>=3.10
 Description-Content-Type: text/markdown

{nemo_evaluator_launcher-0.1.0rc4.dist-info → nemo_evaluator_launcher-0.1.0rc5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nemo_evaluator_launcher/__init__.py,sha256=2F703fttLaIyMHoVD54rptHMXt4AWnplHDrwWJ3e3PM,1930
-nemo_evaluator_launcher/package_info.py,sha256=gsvsf29buE7HrtdCH6Ygr09BputKc65VfnD5z78wuqU,1427
+nemo_evaluator_launcher/package_info.py,sha256=sginTs4w_dL6skrnEgbCWyjmTIN6Vra-t1s46Q86M3I,1427
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
 nemo_evaluator_launcher/api/functional.py,sha256=lsLV0RUvVqd1VG0KAgUDmHPe7DUVFJ5bVaWUuSILx7A,24918
 nemo_evaluator_launcher/api/types.py,sha256=RXr_QoKdhejj1T9-HybSjd4KTxJmSv0bE0uLUFtF7Zc,3269
@@ -16,7 +16,7 @@ nemo_evaluator_launcher/cli/status.py,sha256=JYQnT3YVtx9lzxme4Xak00Cme5MN87wyt4k
 nemo_evaluator_launcher/cli/version.py,sha256=puMwIvkmfD3HESjftdTSP6T3Nc8J4cbz8uXWHJcTemY,2030
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=ZSMClC2kl6KI28tnHCEtNbbFZ2tLKcSbFeUenbb4E6Q,6547
-nemo_evaluator_launcher/common/helpers.py,sha256=UVf_XJv6wmNZ7ftOStOmXpCNZes0SJz6h_X4ZTwoFII,6208
+nemo_evaluator_launcher/common/helpers.py,sha256=eQKNsLp9D5NoKZXOjoaKnu3WJAQF3GCKWOIJldG4sKU,7632
 nemo_evaluator_launcher/common/logging_utils.py,sha256=8UMAQ22t5NAJRDZtI0gVbdKUlNAiG23WQwZZ0HwzOT4,11843
 nemo_evaluator_launcher/common/mapping.py,sha256=tD3jWN7rm9-iJEFlENhYMt7adz8DKs67g3Xd43XIAMM,10731
 nemo_evaluator_launcher/configs/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
@@ -39,7 +39,7 @@ nemo_evaluator_launcher/executors/local/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQn
 nemo_evaluator_launcher/executors/local/executor.py,sha256=ctiyi2rMi01lzTInFdtUdetXU0JTPlT3E-acFjgWdjA,17802
 nemo_evaluator_launcher/executors/local/run.template.sh,sha256=ta55-WukiVXO2hyqqt0JCEjW23JrF41DYG0dBT1tdJA,3320
 nemo_evaluator_launcher/executors/slurm/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
-nemo_evaluator_launcher/executors/slurm/executor.py,sha256=zD3P7fY-QmyZAtDdyTfzdblYMi3bZYaDZAXdDG_Jh-U,35791
+nemo_evaluator_launcher/executors/slurm/executor.py,sha256=xfSvQ1MRlAqM6TI9Rsmh-MNIq0DdHaHo_Vu_SlrMNUM,35813
 nemo_evaluator_launcher/exporters/__init__.py,sha256=mBXG9FG48FeYrs8sF0zA2mgo1eqBmRgoml7zjJrqDso,1323
 nemo_evaluator_launcher/exporters/base.py,sha256=toeitHi-reouJvhRULtsceMlpZat4fHcQIXIbAKury0,3904
 nemo_evaluator_launcher/exporters/gsheets.py,sha256=P2TeHr63qXIGh5nzXhs2l10n5_r2I2C4uWx3pbjX-EY,15084
@@ -49,9 +49,9 @@ nemo_evaluator_launcher/exporters/registry.py,sha256=XsPTv_SBAFjcErO6BJ3OHqs3EvX
 nemo_evaluator_launcher/exporters/utils.py,sha256=uXH4b-Hk7_FQyLOjMRB0b3zK-Ksb2rGlSdc-oECfGHI,24756
 nemo_evaluator_launcher/exporters/wandb.py,sha256=xdaPNw0QM0jZo20UERbViy_vFT-HgbLYzTgmWaev_kk,13430
 nemo_evaluator_launcher/resources/mapping.toml,sha256=uOg4Y-gDXXskbbba2vuwJ5FLJ3W0kSZz7Fap_nJnFQc,11322
-nemo_evaluator_launcher-0.1.0rc4.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.0rc4.dist-info/METADATA,sha256=Qh-pSnJE--iCO90Kv9tGE7SflBaWk-Kp5jm86QfT10g,1282
-nemo_evaluator_launcher-0.1.0rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.0rc4.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
-nemo_evaluator_launcher-0.1.0rc4.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.0rc4.dist-info/RECORD,,
+nemo_evaluator_launcher-0.1.0rc5.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.0rc5.dist-info/METADATA,sha256=EFGSW8P-9TWK3k0p2uhkek2vVP4v2QMVoY5BIxzQAQc,1282
+nemo_evaluator_launcher-0.1.0rc5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.0rc5.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
+nemo_evaluator_launcher-0.1.0rc5.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.0rc5.dist-info/RECORD,,