nemo-evaluator-launcher 0.1.9__py3-none-any.whl → 0.1.10__py3-none-any.whl

nemo_evaluator_launcher/common/helpers.py

@@ -75,7 +75,7 @@ def get_eval_factory_command(
     create_file_cmd = _yaml_to_echo_command(
         yaml.safe_dump(config_fields), "config_ef.yaml"
     )
-    eval_command = f"""eval-factory run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
+    eval_command = f"""cmd=$([[ $(command -v nemo-evaluator) ]] && echo 'nemo-evaluator' || echo 'eval-factory') && $cmd run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
 
     if overrides:
         eval_command = f"{eval_command} --overrides {overrides_str}"

nemo_evaluator_launcher/configs/deployment/trtllm.yaml

@@ -0,0 +1,24 @@
+type: trtllm
+image: nvcr.io/nvidia/tensorrt-llm/release:1.0.0
+checkpoint_path: ???
+served_model_name: ???
+port: 8000
+tensor_parallel_size: 4
+pipeline_parallel_size: 1
+extra_args: ""
+
+endpoints:
+  chat: /v1/chat/completions
+  completions: /v1/completions
+  health: /health
+
+command:
+  mpirun --allow-run-as-root --oversubscribe
+  trtllm-serve serve /checkpoint
+  --tp_size=${deployment.tensor_parallel_size}
+  --pp_size=${deployment.pipeline_parallel_size}
+  --host 0.0.0.0
+  --port ${deployment.port}
+  --backend pytorch
+  --trust_remote_code
+  ${deployment.extra_args}

nemo_evaluator_launcher/executors/local/run.template.sh

@@ -34,7 +34,7 @@ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.pre-start"
 # Docker run with eval factory command
 (
     echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.running"
-    docker run --rm --shm-size=100g \
+    docker run --rm --shm-size=100g --network=host \
       --name {{ task.container_name }} \
       --volume "$artifacts_dir":/results \
       {% for env_var in task.env_vars -%}

nemo_evaluator_launcher/package_info.py

@@ -16,7 +16,7 @@
 # Below is the _next_ version that will be published, not the currently published one.
 MAJOR = 0
 MINOR = 1
-PATCH = 9
+PATCH = 10
 PRE_RELEASE = ""
 
 # Use the following formatting: (major, minor, patch, pre-release)

{nemo_evaluator_launcher-0.1.9.dist-info → nemo_evaluator_launcher-0.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nemo-evaluator-launcher
-Version: 0.1.9
+Version: 0.1.10
 Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
 Author: NVIDIA
 Author-email: nemo-toolkit@nvidia.com

{nemo_evaluator_launcher-0.1.9.dist-info → nemo_evaluator_launcher-0.1.10.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nemo_evaluator_launcher/__init__.py,sha256=2F703fttLaIyMHoVD54rptHMXt4AWnplHDrwWJ3e3PM,1930
-nemo_evaluator_launcher/package_info.py,sha256=vMYywsXKM5Otm0BHGYo9IfNmXKclKT4swGz0uymoMMU,1585
+nemo_evaluator_launcher/package_info.py,sha256=JxMCBGf9BEWOIblJPlJOASEOhjxJf4WyCT-x4CAdtNQ,1586
 nemo_evaluator_launcher/api/__init__.py,sha256=U9q_MJK2vRsFaymanhyy0nD1SNAZQZC8oY45RXPX7ac,1024
 nemo_evaluator_launcher/api/functional.py,sha256=6QuCVOkox4SAxtNQhEBzzsMTHaTwHCTLFRqz6eC0gDY,27053
 nemo_evaluator_launcher/api/types.py,sha256=RXr_QoKdhejj1T9-HybSjd4KTxJmSv0bE0uLUFtF7Zc,3269
@@ -15,7 +15,7 @@ nemo_evaluator_launcher/cli/status.py,sha256=QCZfXQVnQQKNyyVWJtQd1Xe9uhl5YebfhSe
 nemo_evaluator_launcher/cli/version.py,sha256=puMwIvkmfD3HESjftdTSP6T3Nc8J4cbz8uXWHJcTemY,2030
 nemo_evaluator_launcher/common/__init__.py,sha256=6-xb4KpG8-lZbWBI42c_Gax-Sq0kMSW8UG0Vn8dOBlo,744
 nemo_evaluator_launcher/common/execdb.py,sha256=WPzg5Iu2ojvFpBuYahSt3voP_iEUpoO8NgqMLUBwFxA,9767
-nemo_evaluator_launcher/common/helpers.py,sha256=55RSURjb6SUrK-Pc8oqiYuqlL5JFYk2B-UTlX85Vzxs,7625
+nemo_evaluator_launcher/common/helpers.py,sha256=YG1mQNijKZkw7mqTPhU3yatgWzufXyyKc6oSxuhJpLs,7710
 nemo_evaluator_launcher/common/logging_utils.py,sha256=8UMAQ22t5NAJRDZtI0gVbdKUlNAiG23WQwZZ0HwzOT4,11843
 nemo_evaluator_launcher/common/mapping.py,sha256=tD3jWN7rm9-iJEFlENhYMt7adz8DKs67g3Xd43XIAMM,10731
 nemo_evaluator_launcher/configs/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
@@ -24,6 +24,7 @@ nemo_evaluator_launcher/configs/deployment/generic.yaml,sha256=8_Z0fcjZuH6GfV9jJ
 nemo_evaluator_launcher/configs/deployment/nim.yaml,sha256=hRJGwCR2XIS3bUFWkXzmroLep81KaHa3Sn2_edfWbkU,1266
 nemo_evaluator_launcher/configs/deployment/none.yaml,sha256=buPada1yMz9ClPDbq63vPDzLGr_IubTLia91DG3i5Lo,684
 nemo_evaluator_launcher/configs/deployment/sglang.yaml,sha256=2CrnmpdEr5HvUbZW6k_wFPK41q6e4WoS7CJKAcgKlck,1273
+nemo_evaluator_launcher/configs/deployment/trtllm.yaml,sha256=nZF1ueCF9uMticElgePPy8ecq2vbfvWxj_5YwQTJ--k,570
 nemo_evaluator_launcher/configs/deployment/vllm.yaml,sha256=Z3nhybadfB28fqlNz-ihALJC3PfzC_ZF_BecytGeVR0,1415
 nemo_evaluator_launcher/configs/execution/local.yaml,sha256=Hh8yhKdqihDORGuMd8NbXXTmNM5SlfdR21PKEpiuVpk,701
 nemo_evaluator_launcher/configs/execution/lepton/default.yaml,sha256=SRSMxohEtafzb-QS_oz4kP-RBgigRCAZgYwKkXtjymY,2930
@@ -37,7 +38,7 @@ nemo_evaluator_launcher/executors/lepton/executor.py,sha256=37ZjRtu69Mff1o9dXSI_
 nemo_evaluator_launcher/executors/lepton/job_helpers.py,sha256=6baTxcygfP1oFgAJ7I9EL4xRlcJDWqbqzZoE1CRrwSk,13528
 nemo_evaluator_launcher/executors/local/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/executors/local/executor.py,sha256=wM7DKLncDMDCOQFlQUEdb8KViL_WkTXi0kpCm-fnP4M,17771
-nemo_evaluator_launcher/executors/local/run.template.sh,sha256=ta55-WukiVXO2hyqqt0JCEjW23JrF41DYG0dBT1tdJA,3320
+nemo_evaluator_launcher/executors/local/run.template.sh,sha256=s6sqA9fx9d80qC56i1A7rnHMSe397qK6SKmt2TvLRNk,3335
 nemo_evaluator_launcher/executors/slurm/__init__.py,sha256=lNC_skFLYTOt-arnY3ZQnZMWzHlrtD2wAoHvDcHddwM,673
 nemo_evaluator_launcher/executors/slurm/executor.py,sha256=eG0hR0BuA6KRWMCaqtx2dhKeEZgm1QCC_edaEy32kDs,36274
 nemo_evaluator_launcher/exporters/__init__.py,sha256=mBXG9FG48FeYrs8sF0zA2mgo1eqBmRgoml7zjJrqDso,1323
@@ -49,9 +50,9 @@ nemo_evaluator_launcher/exporters/registry.py,sha256=XsPTv_SBAFjcErO6BJ3OHqs3EvX
 nemo_evaluator_launcher/exporters/utils.py,sha256=uXH4b-Hk7_FQyLOjMRB0b3zK-Ksb2rGlSdc-oECfGHI,24756
 nemo_evaluator_launcher/exporters/wandb.py,sha256=xdaPNw0QM0jZo20UERbViy_vFT-HgbLYzTgmWaev_kk,13430
 nemo_evaluator_launcher/resources/mapping.toml,sha256=uOg4Y-gDXXskbbba2vuwJ5FLJ3W0kSZz7Fap_nJnFQc,11322
-nemo_evaluator_launcher-0.1.9.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
-nemo_evaluator_launcher-0.1.9.dist-info/METADATA,sha256=zgufAH2WXM7f6eXkV_vtKaYJhwom3GZ8oO4-UYKvCE8,28724
-nemo_evaluator_launcher-0.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nemo_evaluator_launcher-0.1.9.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
-nemo_evaluator_launcher-0.1.9.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
-nemo_evaluator_launcher-0.1.9.dist-info/RECORD,,
+nemo_evaluator_launcher-0.1.10.dist-info/licenses/LICENSE,sha256=DyGb0fqHPZAsd_uXHA0DGcOCqsvrNsImuLC0Ts4s1zI,23413
+nemo_evaluator_launcher-0.1.10.dist-info/METADATA,sha256=wljBwSTRO124xl4yj8S7wq4K5OBte1Wx1Mzx57fD4Ts,28725
+nemo_evaluator_launcher-0.1.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nemo_evaluator_launcher-0.1.10.dist-info/entry_points.txt,sha256=64z1T5GKSB9PW1fCENQuor6X6eqH1rcfg0NQGfKrEy8,130
+nemo_evaluator_launcher-0.1.10.dist-info/top_level.txt,sha256=5PvawNm9TXKqPRjZita1xPOtFiMOipcoRf50FI1iY3s,24
+nemo_evaluator_launcher-0.1.10.dist-info/RECORD,,