nemo-evaluator-launcher 0.1.9__tar.gz → 0.1.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of nemo-evaluator-launcher might be problematic. Click here for more details.

Files changed (63) hide show
  1. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/PKG-INFO +1 -1
  2. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/common/helpers.py +1 -1
  3. nemo_evaluator_launcher-0.1.11/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml +24 -0
  4. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/local/run.template.sh +1 -1
  5. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/package_info.py +1 -1
  6. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/PKG-INFO +1 -1
  7. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt +1 -0
  8. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/LICENSE +0 -0
  9. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/README.md +0 -0
  10. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/pyproject.toml +0 -0
  11. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/setup.cfg +0 -0
  12. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/__init__.py +0 -0
  13. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/api/__init__.py +0 -0
  14. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/api/functional.py +0 -0
  15. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/api/types.py +0 -0
  16. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/api/utils.py +0 -0
  17. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/__init__.py +0 -0
  18. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/export.py +0 -0
  19. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/kill.py +0 -0
  20. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/ls_runs.py +0 -0
  21. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/ls_tasks.py +0 -0
  22. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/main.py +0 -0
  23. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/run.py +0 -0
  24. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/status.py +0 -0
  25. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/cli/version.py +0 -0
  26. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/common/__init__.py +0 -0
  27. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/common/execdb.py +0 -0
  28. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/common/logging_utils.py +0 -0
  29. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/common/mapping.py +0 -0
  30. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/__init__.py +0 -0
  31. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/default.yaml +0 -0
  32. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/deployment/generic.yaml +0 -0
  33. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/deployment/nim.yaml +0 -0
  34. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/deployment/none.yaml +0 -0
  35. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/deployment/sglang.yaml +0 -0
  36. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/deployment/vllm.yaml +0 -0
  37. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml +0 -0
  38. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/execution/local.yaml +0 -0
  39. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/configs/execution/slurm/default.yaml +0 -0
  40. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/__init__.py +0 -0
  41. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/base.py +0 -0
  42. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/lepton/__init__.py +0 -0
  43. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/lepton/deployment_helpers.py +0 -0
  44. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/lepton/executor.py +0 -0
  45. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/lepton/job_helpers.py +0 -0
  46. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/local/__init__.py +0 -0
  47. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/local/executor.py +0 -0
  48. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/registry.py +0 -0
  49. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/slurm/__init__.py +0 -0
  50. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/slurm/executor.py +0 -0
  51. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/__init__.py +0 -0
  52. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/base.py +0 -0
  53. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/gsheets.py +0 -0
  54. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/local.py +0 -0
  55. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/mlflow.py +0 -0
  56. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/registry.py +0 -0
  57. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/utils.py +0 -0
  58. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/exporters/wandb.py +0 -0
  59. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/resources/mapping.toml +0 -0
  60. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/dependency_links.txt +0 -0
  61. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/entry_points.txt +0 -0
  62. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/requires.txt +0 -0
  63. {nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/top_level.txt +0 -0
{nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nemo-evaluator-launcher
3
- Version: 0.1.9
3
+ Version: 0.1.11
4
4
  Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
5
5
  Author: NVIDIA
6
6
  Author-email: nemo-toolkit@nvidia.com
{nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/common/helpers.py RENAMED
@@ -75,7 +75,7 @@ def get_eval_factory_command(
75
75
  create_file_cmd = _yaml_to_echo_command(
76
76
  yaml.safe_dump(config_fields), "config_ef.yaml"
77
77
  )
78
- eval_command = f"""eval-factory run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
78
+ eval_command = f"""cmd=$([[ $(command -v nemo-evaluator) ]] && echo 'nemo-evaluator' || echo 'eval-factory') && $cmd run_eval --model_id {model_id} --model_type {model_type} --eval_type {eval_type} --model_url {model_url} --api_key_name API_KEY --output_dir /results --run_config config_ef.yaml"""
79
79
 
80
80
  if overrides:
81
81
  eval_command = f"{eval_command} --overrides {overrides_str}"
nemo_evaluator_launcher-0.1.11/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml ADDED
@@ -0,0 +1,24 @@
1
+ type: trtllm
2
+ image: nvcr.io/nvidia/tensorrt-llm/release:1.0.0
3
+ checkpoint_path: ???
4
+ served_model_name: ???
5
+ port: 8000
6
+ tensor_parallel_size: 4
7
+ pipeline_parallel_size: 1
8
+ extra_args: ""
9
+
10
+ endpoints:
11
+ chat: /v1/chat/completions
12
+ completions: /v1/completions
13
+ health: /health
14
+
15
+ command:
16
+ mpirun --allow-run-as-root --oversubscribe
17
+ trtllm-serve serve /checkpoint
18
+ --tp_size=${deployment.tensor_parallel_size}
19
+ --pp_size=${deployment.pipeline_parallel_size}
20
+ --host 0.0.0.0
21
+ --port ${deployment.port}
22
+ --backend pytorch
23
+ --trust_remote_code
24
+ ${deployment.extra_args}
{nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/executors/local/run.template.sh RENAMED
@@ -34,7 +34,7 @@ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.pre-start"
34
34
  # Docker run with eval factory command
35
35
  (
36
36
  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$logs_dir/stage.running"
37
- docker run --rm --shm-size=100g \
37
+ docker run --rm --shm-size=100g --network=host \
38
38
  --name {{ task.container_name }} \
39
39
  --volume "$artifacts_dir":/results \
40
40
  {% for env_var in task.env_vars -%}
{nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher/package_info.py RENAMED
@@ -16,7 +16,7 @@
16
16
  # Below is the _next_ version that will be published, not the currently published one.
17
17
  MAJOR = 0
18
18
  MINOR = 1
19
- PATCH = 9
19
+ PATCH = 11
20
20
  PRE_RELEASE = ""
21
21
 
22
22
  # Use the following formatting: (major, minor, patch, pre-release)
{nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nemo-evaluator-launcher
3
- Version: 0.1.9
3
+ Version: 0.1.11
4
4
  Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
5
5
  Author: NVIDIA
6
6
  Author-email: nemo-toolkit@nvidia.com
{nemo_evaluator_launcher-0.1.9 → nemo_evaluator_launcher-0.1.11}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt RENAMED
@@ -33,6 +33,7 @@ src/nemo_evaluator_launcher/configs/deployment/generic.yaml
33
33
  src/nemo_evaluator_launcher/configs/deployment/nim.yaml
34
34
  src/nemo_evaluator_launcher/configs/deployment/none.yaml
35
35
  src/nemo_evaluator_launcher/configs/deployment/sglang.yaml
36
+ src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml
36
37
  src/nemo_evaluator_launcher/configs/deployment/vllm.yaml
37
38
  src/nemo_evaluator_launcher/configs/execution/local.yaml
38
39
  src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml