nemo-evaluator-launcher 0.1.21__tar.gz → 0.1.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of nemo-evaluator-launcher might be problematic. Click here for more details.

Files changed (65) hide show
  1. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/PKG-INFO +2 -2
  2. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/pyproject.toml +1 -1
  3. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/package_info.py +1 -1
  4. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/resources/mapping.toml +12 -12
  5. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/PKG-INFO +2 -2
  6. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/requires.txt +1 -1
  7. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/LICENSE +0 -0
  8. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/README.md +0 -0
  9. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/setup.cfg +0 -0
  10. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/__init__.py +0 -0
  11. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/api/__init__.py +0 -0
  12. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/api/functional.py +0 -0
  13. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/api/types.py +0 -0
  14. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/api/utils.py +0 -0
  15. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/__init__.py +0 -0
  16. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/export.py +0 -0
  17. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/info.py +0 -0
  18. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/kill.py +0 -0
  19. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/ls_runs.py +0 -0
  20. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/ls_tasks.py +0 -0
  21. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/main.py +0 -0
  22. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/run.py +0 -0
  23. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/status.py +0 -0
  24. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/cli/version.py +0 -0
  25. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/common/__init__.py +0 -0
  26. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/common/execdb.py +0 -0
  27. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/common/helpers.py +0 -0
  28. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/common/logging_utils.py +0 -0
  29. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/common/mapping.py +0 -0
  30. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/common/printing_utils.py +0 -0
  31. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/__init__.py +0 -0
  32. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/default.yaml +0 -0
  33. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/deployment/generic.yaml +0 -0
  34. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/deployment/nim.yaml +0 -0
  35. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/deployment/none.yaml +0 -0
  36. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/deployment/sglang.yaml +0 -0
  37. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml +0 -0
  38. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/deployment/vllm.yaml +0 -0
  39. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml +0 -0
  40. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/execution/local.yaml +0 -0
  41. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/configs/execution/slurm/default.yaml +0 -0
  42. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/__init__.py +0 -0
  43. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/base.py +0 -0
  44. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/lepton/__init__.py +0 -0
  45. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/lepton/deployment_helpers.py +0 -0
  46. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/lepton/executor.py +0 -0
  47. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/lepton/job_helpers.py +0 -0
  48. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/local/__init__.py +0 -0
  49. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/local/executor.py +0 -0
  50. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/local/run.template.sh +0 -0
  51. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/registry.py +0 -0
  52. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/slurm/__init__.py +0 -0
  53. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/executors/slurm/executor.py +0 -0
  54. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/__init__.py +0 -0
  55. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/base.py +0 -0
  56. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/gsheets.py +0 -0
  57. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/local.py +0 -0
  58. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/mlflow.py +0 -0
  59. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/registry.py +0 -0
  60. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/utils.py +0 -0
  61. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/exporters/wandb.py +0 -0
  62. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/SOURCES.txt +0 -0
  63. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/dependency_links.txt +0 -0
  64. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/entry_points.txt +0 -0
  65. {nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/top_level.txt +0 -0
{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nemo-evaluator-launcher
3
- Version: 0.1.21
3
+ Version: 0.1.22
4
4
  Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
5
5
  Author: NVIDIA
6
6
  Author-email: nemo-toolkit@nvidia.com
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
478
478
  Provides-Extra: wandb
479
479
  Requires-Dist: wandb>=0.15.0; extra == "wandb"
480
480
  Provides-Extra: gsheets
481
- Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
481
+ Requires-Dist: gspread>=5.0.0; extra == "gsheets"
482
482
  Provides-Extra: exporters
483
483
  Requires-Dist: mlflow; extra == "exporters"
484
484
  Requires-Dist: wandb; extra == "exporters"
{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/pyproject.toml RENAMED
@@ -40,7 +40,7 @@ repository = "https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator-l
40
40
  [project.optional-dependencies]
41
41
  mlflow = ["mlflow>=2.8.0"]
42
42
  wandb = ["wandb>=0.15.0"]
43
- gsheets = ["gsheets>=0.1.0"]
43
+ gsheets = ["gspread>=5.0.0"]
44
44
  exporters = ["mlflow", "wandb", "gsheets"]
45
45
  all = ["mlflow", "wandb", "gsheets"]
46
46
 
{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/package_info.py RENAMED
@@ -16,7 +16,7 @@
16
16
  # Below is the _next_ version that will be published, not the currently published one.
17
17
  MAJOR = 0
18
18
  MINOR = 1
19
- PATCH = 21
19
+ PATCH = 22
20
20
  PRE_RELEASE = ""
21
21
 
22
22
  # Use the following formatting: (major, minor, patch, pre-release)
{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher/resources/mapping.toml RENAMED
@@ -304,29 +304,29 @@ required_env_vars = []
304
304
 
305
305
  ###############################################################################
306
306
  # NOTE(wprazuch): to verify if the tasks need any env var setting
307
- [nemo-skills]
307
+ [nemo_skills]
308
308
  container = "nvcr.io/nvidia/eval-factory/nemo_skills:25.09.1"
309
309
 
310
- [nemo-skills.tasks.chat.ns_aime2024]
311
- required_env_vars = []
310
+ [nemo_skills.tasks.chat.ns_aime2024]
311
+ required_env_vars = ["JUDGE_API_KEY"]
312
312
 
313
- [nemo-skills.tasks.chat.ns_aime2025]
313
+ [nemo_skills.tasks.chat.ns_aime2025]
314
314
  required_env_vars = []
315
315
 
316
- [nemo-skills.tasks.chat.ns_bfcl_v3]
316
+ [nemo_skills.tasks.chat.ns_bfcl_v3]
317
317
  required_env_vars = []
318
318
 
319
- [nemo-skills.tasks.chat.ns_gpqa]
320
- required_env_vars = []
319
+ [nemo_skills.tasks.chat.ns_gpqa]
320
+ required_env_vars = ["HF_TOKEN"]
321
321
 
322
- [nemo-skills.tasks.chat.ns_hle]
322
+ [nemo_skills.tasks.chat.ns_hle]
323
323
  required_env_vars = []
324
324
 
325
- [nemo-skills.tasks.chat.ns_mmlu]
326
- required_env_vars = []
325
+ [nemo_skills.tasks.chat.ns_mmlu]
326
+ required_env_vars = ["HF_TOKEN"]
327
327
 
328
- [nemo-skills.tasks.chat.ns_mmlu_pro]
329
- required_env_vars = []
328
+ [nemo_skills.tasks.chat.ns_mmlu_pro]
329
+ required_env_vars = ["HF_TOKEN"]
330
330
 
331
331
  ###############################################################################
332
332
  [safety-harness]
{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nemo-evaluator-launcher
3
- Version: 0.1.21
3
+ Version: 0.1.22
4
4
  Summary: Launcher for the evaluations provided by NeMo Evaluator containers with different runtime backends
5
5
  Author: NVIDIA
6
6
  Author-email: nemo-toolkit@nvidia.com
@@ -478,7 +478,7 @@ Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
478
478
  Provides-Extra: wandb
479
479
  Requires-Dist: wandb>=0.15.0; extra == "wandb"
480
480
  Provides-Extra: gsheets
481
- Requires-Dist: gsheets>=0.1.0; extra == "gsheets"
481
+ Requires-Dist: gspread>=5.0.0; extra == "gsheets"
482
482
  Provides-Extra: exporters
483
483
  Requires-Dist: mlflow; extra == "exporters"
484
484
  Requires-Dist: wandb; extra == "exporters"
{nemo_evaluator_launcher-0.1.21 → nemo_evaluator_launcher-0.1.22}/src/nemo_evaluator_launcher.egg-info/requires.txt RENAMED
@@ -21,7 +21,7 @@ wandb
21
21
  gsheets
22
22
 
23
23
  [gsheets]
24
- gsheets>=0.1.0
24
+ gspread>=5.0.0
25
25
 
26
26
  [mlflow]
27
27
  mlflow>=2.8.0