nemo-evaluator-launcher 0.1.22__tar.gz → 0.1.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of nemo-evaluator-launcher might be problematic. Click here for more details.

Files changed (65) hide show
  1. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/PKG-INFO +1 -1
  2. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/helpers.py +4 -1
  3. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/package_info.py +1 -1
  4. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/resources/mapping.toml +1 -1
  5. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/PKG-INFO +1 -1
  6. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/LICENSE +0 -0
  7. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/README.md +0 -0
  8. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/pyproject.toml +0 -0
  9. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/setup.cfg +0 -0
  10. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/__init__.py +0 -0
  11. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/api/__init__.py +0 -0
  12. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/api/functional.py +0 -0
  13. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/api/types.py +0 -0
  14. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/api/utils.py +0 -0
  15. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/__init__.py +0 -0
  16. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/export.py +0 -0
  17. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/info.py +0 -0
  18. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/kill.py +0 -0
  19. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/ls_runs.py +0 -0
  20. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/ls_tasks.py +0 -0
  21. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/main.py +0 -0
  22. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/run.py +0 -0
  23. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/status.py +0 -0
  24. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/cli/version.py +0 -0
  25. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/__init__.py +0 -0
  26. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/execdb.py +0 -0
  27. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/logging_utils.py +0 -0
  28. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/mapping.py +0 -0
  29. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/printing_utils.py +0 -0
  30. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/__init__.py +0 -0
  31. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/default.yaml +0 -0
  32. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/deployment/generic.yaml +0 -0
  33. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/deployment/nim.yaml +0 -0
  34. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/deployment/none.yaml +0 -0
  35. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/deployment/sglang.yaml +0 -0
  36. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml +0 -0
  37. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/deployment/vllm.yaml +0 -0
  38. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml +0 -0
  39. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/execution/local.yaml +0 -0
  40. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/configs/execution/slurm/default.yaml +0 -0
  41. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/__init__.py +0 -0
  42. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/base.py +0 -0
  43. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/lepton/__init__.py +0 -0
  44. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/lepton/deployment_helpers.py +0 -0
  45. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/lepton/executor.py +0 -0
  46. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/lepton/job_helpers.py +0 -0
  47. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/local/__init__.py +0 -0
  48. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/local/executor.py +0 -0
  49. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/local/run.template.sh +0 -0
  50. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/registry.py +0 -0
  51. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/slurm/__init__.py +0 -0
  52. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/executors/slurm/executor.py +0 -0
  53. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/__init__.py +0 -0
  54. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/base.py +0 -0
  55. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/gsheets.py +0 -0
  56. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/local.py +0 -0
  57. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/mlflow.py +0 -0
  58. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/registry.py +0 -0
  59. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/utils.py +0 -0
  60. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/exporters/wandb.py +0 -0
  61. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt +0 -0
  62. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/dependency_links.txt +0 -0
  63. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/entry_points.txt +0 -0
  64. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/requires.txt +0 -0
  65. {nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/top_level.txt +0 -0
{nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nemo-evaluator-launcher
3
- Version: 0.1.22
3
+ Version: 0.1.23
4
4
  Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
5
5
  Author: NVIDIA
6
6
  Author-email: nemo-toolkit@nvidia.com
{nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/common/helpers.py RENAMED
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
  import base64
17
+ import copy
17
18
  import datetime
18
19
  from dataclasses import dataclass
19
20
  from typing import Optional
@@ -170,7 +171,9 @@ def get_eval_factory_command(
170
171
  + "&& $cmd run_eval --run_config config_ef.yaml"
171
172
  )
172
173
 
173
- overrides = merged_nemo_evaluator_config.get("overrides", {})
174
+ # NOTE: see note and test about deprecating that.
175
+ overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
176
+ overrides.update(dict(user_task_config.get("overrides", {})))
174
177
  # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
175
178
  # Consider passing a JSON object on Eval Factory side
176
179
  overrides = {
{nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/package_info.py RENAMED
@@ -16,7 +16,7 @@
16
16
  # Below is the _next_ version that will be published, not the currently published one.
17
17
  MAJOR = 0
18
18
  MINOR = 1
19
- PATCH = 22
19
+ PATCH = 23
20
20
  PRE_RELEASE = ""
21
21
 
22
22
  # Use the following formatting: (major, minor, patch, pre-release)
{nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher/resources/mapping.toml RENAMED
@@ -226,7 +226,7 @@ required_env_vars = []
226
226
  [bigcode-evaluation-harness.tasks.completions.humaneval]
227
227
  required_env_vars = []
228
228
 
229
- [bigcode-evaluation-harness.tasks.completions.humaneval_instruct]
229
+ [bigcode-evaluation-harness.tasks.chat.humaneval_instruct]
230
230
 
231
231
 
232
232
  ###############################################################################
{nemo_evaluator_launcher-0.1.22 → nemo_evaluator_launcher-0.1.23}/src/nemo_evaluator_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nemo-evaluator-launcher
3
- Version: 0.1.22
3
+ Version: 0.1.23
4
4
  Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
5
5
  Author: NVIDIA
6
6
  Author-email: nemo-toolkit@nvidia.com