crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crfm-helm might be problematic. Click here for more details.

Files changed (268) hide show
  1. {crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/METADATA +74 -53
  2. {crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/RECORD +262 -182
  3. {crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/WHEEL +1 -1
  4. helm/benchmark/adaptation/adapters/in_context_learning_adapter.py +3 -3
  5. helm/benchmark/adaptation/adapters/test_adapter.py +4 -4
  6. helm/benchmark/annotation/air_bench_annotator.py +2 -2
  7. helm/benchmark/annotation/bigcodebench_annotator.py +3 -3
  8. helm/benchmark/annotation/bird_sql_annotator.py +2 -2
  9. helm/benchmark/annotation/chw_care_plan_annotator.py +7 -12
  10. helm/benchmark/annotation/ehr_sql_annotator.py +2 -2
  11. helm/benchmark/annotation/helpdesk_call_summarization_annotator.py +7 -7
  12. helm/benchmark/annotation/live_qa_annotator.py +1 -1
  13. helm/benchmark/annotation/mimic_bhc_annotator.py +100 -0
  14. helm/benchmark/annotation/model_as_judge.py +12 -16
  15. helm/benchmark/annotation/omni_math_annotator.py +13 -14
  16. helm/benchmark/annotation/wildbench_annotator.py +9 -9
  17. helm/benchmark/executor.py +11 -12
  18. helm/benchmark/metrics/aci_bench_metrics.py +9 -29
  19. helm/benchmark/metrics/bias_word_lists.py +1 -1
  20. helm/benchmark/metrics/chw_care_plan_metrics.py +10 -30
  21. helm/benchmark/metrics/classification_metrics.py +3 -3
  22. helm/benchmark/metrics/cleva_harms_metrics.py +2 -2
  23. helm/benchmark/metrics/codeinsights_code_efficiency_metrics.py +186 -0
  24. helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py +477 -0
  25. helm/benchmark/metrics/codeinsights_correct_code_metrics.py +366 -0
  26. helm/benchmark/metrics/codeinsights_edge_case_metrics.py +92 -0
  27. helm/benchmark/metrics/codeinsights_metric_specs.py +51 -0
  28. helm/benchmark/metrics/comet_metric.py +1 -1
  29. helm/benchmark/metrics/conv_fin_qa_calc_metrics.py +2 -2
  30. helm/benchmark/metrics/copyright_metrics.py +1 -1
  31. helm/benchmark/metrics/decodingtrust_stereotype_bias_metrics.py +1 -1
  32. helm/benchmark/metrics/dischargeme_metrics.py +9 -29
  33. helm/benchmark/metrics/efficiency_metrics.py +3 -3
  34. helm/benchmark/metrics/evaluate_reference_metrics.py +1 -1
  35. helm/benchmark/metrics/gpt4_audio_refusal_metrics.py +145 -0
  36. helm/benchmark/metrics/ifeval_metrics.py +2 -2
  37. helm/benchmark/metrics/image_generation/clip_score_metrics.py +13 -2
  38. helm/benchmark/metrics/image_generation/fractal_dimension/fractal_dimension_util.py +1 -1
  39. helm/benchmark/metrics/kpi_edgar_metrics.py +121 -0
  40. helm/benchmark/metrics/llm_jury_metrics.py +46 -0
  41. helm/benchmark/metrics/lmkt_metric_specs.py +12 -0
  42. helm/benchmark/metrics/lmkt_metrics.py +47 -0
  43. helm/benchmark/metrics/med_dialog_metrics.py +9 -29
  44. helm/benchmark/metrics/medalign_metrics.py +9 -29
  45. helm/benchmark/metrics/medi_qa_metrics.py +9 -29
  46. helm/benchmark/metrics/medication_qa_metrics.py +10 -30
  47. helm/benchmark/metrics/melt_bias_metric.py +234 -0
  48. helm/benchmark/metrics/melt_bias_word_lists.py +1367 -0
  49. helm/benchmark/metrics/melt_metric_specs.py +43 -0
  50. helm/benchmark/metrics/melt_toxicity_metric.py +107 -0
  51. helm/benchmark/metrics/mental_health_metrics.py +9 -29
  52. helm/benchmark/metrics/metric_service.py +11 -11
  53. helm/benchmark/metrics/mimic_bhc_metrics.py +14 -0
  54. helm/benchmark/metrics/mimic_rrs_metrics.py +9 -29
  55. helm/benchmark/metrics/mtsamples_procedures_metrics.py +9 -29
  56. helm/benchmark/metrics/mtsamples_replicate_metrics.py +9 -29
  57. helm/benchmark/metrics/openai_mrcr_metrics.py +52 -0
  58. helm/benchmark/metrics/ruler_qa_metrics.py +34 -0
  59. helm/benchmark/metrics/starr_patient_instructions_metrics.py +9 -29
  60. helm/benchmark/metrics/summac/model_summac.py +2 -3
  61. helm/benchmark/metrics/summarization_metrics.py +2 -1
  62. helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py +2 -2
  63. helm/benchmark/metrics/toxicity_metrics.py +2 -2
  64. helm/benchmark/metrics/unitxt_metrics.py +3 -4
  65. helm/benchmark/metrics/vision_language/emd_utils.py +4 -2
  66. helm/benchmark/metrics/vision_language/image_utils.py +2 -2
  67. helm/benchmark/model_deployment_registry.py +16 -26
  68. helm/benchmark/presentation/contamination.py +3 -3
  69. helm/benchmark/presentation/create_plots.py +43 -13
  70. helm/benchmark/presentation/run_display.py +13 -0
  71. helm/benchmark/presentation/schema.py +7 -1
  72. helm/benchmark/presentation/summarize.py +84 -61
  73. helm/benchmark/presentation/test_create_plots.py +4 -1
  74. helm/benchmark/reeval_run.py +3 -4
  75. helm/benchmark/reeval_runner.py +3 -3
  76. helm/benchmark/run.py +84 -73
  77. helm/benchmark/run_expander.py +12 -1
  78. helm/benchmark/run_spec_factory.py +7 -6
  79. helm/benchmark/run_specs/arabic_run_specs.py +73 -0
  80. helm/benchmark/run_specs/audio_run_specs.py +52 -8
  81. helm/benchmark/run_specs/bluex_run_specs.py +40 -0
  82. helm/benchmark/run_specs/classic_run_specs.py +0 -53
  83. helm/benchmark/run_specs/codeinsights_run_specs.py +192 -0
  84. helm/benchmark/run_specs/enterprise_run_specs.py +20 -0
  85. helm/benchmark/run_specs/experimental_run_specs.py +31 -1
  86. helm/benchmark/run_specs/healthqa_br_run_specs.py +40 -0
  87. helm/benchmark/run_specs/heim_run_specs.py +3 -1
  88. helm/benchmark/run_specs/lmkt_run_specs.py +144 -0
  89. helm/benchmark/run_specs/long_context_run_specs.py +114 -15
  90. helm/benchmark/run_specs/medhelm_run_specs.py +146 -41
  91. helm/benchmark/run_specs/melt_run_specs.py +783 -0
  92. helm/benchmark/run_specs/multilingual_run_specs.py +50 -0
  93. helm/benchmark/run_specs/speech_disorder_audio_run_specs.py +163 -0
  94. helm/benchmark/run_specs/vlm_run_specs.py +28 -0
  95. helm/benchmark/runner.py +5 -5
  96. helm/benchmark/scenarios/aci_bench_scenario.py +7 -1
  97. helm/benchmark/scenarios/alghafa_scenario.py +126 -0
  98. helm/benchmark/scenarios/arabic_mmlu_scenario.py +78 -0
  99. helm/benchmark/scenarios/aratrust_scenario.py +76 -0
  100. helm/benchmark/scenarios/audio_language/air_bench_chat_scenario.py +3 -1
  101. helm/benchmark/scenarios/audio_language/air_bench_foundation_scenario.py +5 -5
  102. helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py +1 -1
  103. helm/benchmark/scenarios/audio_language/corebench_scenario.py +77 -0
  104. helm/benchmark/scenarios/audio_language/mustard_scenario.py +1 -1
  105. helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification_scenario.py +104 -0
  106. helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py +99 -0
  107. helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py +118 -0
  108. helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py +86 -0
  109. helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py +117 -0
  110. helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py +15 -1
  111. helm/benchmark/scenarios/audio_language/voxceleb2_scenario.py +1 -2
  112. helm/benchmark/scenarios/autobencher_capabilities_scenario.py +2 -2
  113. helm/benchmark/scenarios/bluex_scenario.py +66 -0
  114. helm/benchmark/scenarios/chw_care_plan_scenario.py +14 -13
  115. helm/benchmark/scenarios/clear_scenario.py +11 -7
  116. helm/benchmark/scenarios/cleva_scenario.py +1 -1
  117. helm/benchmark/scenarios/codeinsights_code_efficiency_scenario.py +197 -0
  118. helm/benchmark/scenarios/codeinsights_correct_code_scenario.py +78 -0
  119. helm/benchmark/scenarios/codeinsights_edge_case_scenario.py +192 -0
  120. helm/benchmark/scenarios/codeinsights_student_coding_scenario.py +162 -0
  121. helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py +188 -0
  122. helm/benchmark/scenarios/dischargeme_scenario.py +36 -21
  123. helm/benchmark/scenarios/ehr_sql_scenario.py +7 -1
  124. helm/benchmark/scenarios/ehrshot_scenario.py +28 -55
  125. helm/benchmark/scenarios/exams_multilingual_scenario.py +115 -0
  126. helm/benchmark/scenarios/grammar.py +2 -2
  127. helm/benchmark/scenarios/headqa_scenario.py +6 -1
  128. helm/benchmark/scenarios/healthqa_br_scenario.py +80 -0
  129. helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py +90 -0
  130. helm/benchmark/scenarios/infinite_bench_en_qa_scenario.py +85 -0
  131. helm/benchmark/scenarios/{infinite_bench_sum_scenario.py → infinite_bench_en_sum_scenario.py} +10 -13
  132. helm/benchmark/scenarios/kpi_edgar_scenario.py +151 -0
  133. helm/benchmark/scenarios/lmkt_scenarios.py +288 -0
  134. helm/benchmark/scenarios/math_scenario.py +21 -20
  135. helm/benchmark/scenarios/med_dialog_scenario.py +6 -1
  136. helm/benchmark/scenarios/medalign_scenario.py +9 -3
  137. helm/benchmark/scenarios/medalign_scenario_helper.py +27 -130
  138. helm/benchmark/scenarios/medbullets_scenario.py +7 -2
  139. helm/benchmark/scenarios/medcalc_bench_scenario.py +4 -2
  140. helm/benchmark/scenarios/medec_scenario.py +6 -1
  141. helm/benchmark/scenarios/medhallu_scenario.py +7 -1
  142. helm/benchmark/scenarios/medi_qa_scenario.py +10 -4
  143. helm/benchmark/scenarios/medication_qa_scenario.py +7 -1
  144. helm/benchmark/scenarios/melt_ir_scenario.py +171 -0
  145. helm/benchmark/scenarios/melt_knowledge_scenario.py +246 -0
  146. helm/benchmark/scenarios/melt_lm_scenarios.py +252 -0
  147. helm/benchmark/scenarios/melt_scenarios.py +793 -0
  148. helm/benchmark/scenarios/melt_srn_scenario.py +342 -0
  149. helm/benchmark/scenarios/melt_synthetic_reasoning_scenario.py +222 -0
  150. helm/benchmark/scenarios/melt_translation_scenario.py +152 -0
  151. helm/benchmark/scenarios/mental_health_scenario.py +16 -5
  152. helm/benchmark/scenarios/mimic_bhc_scenario.py +13 -8
  153. helm/benchmark/scenarios/mimic_rrs_scenario.py +17 -8
  154. helm/benchmark/scenarios/mimiciv_billing_code_scenario.py +14 -8
  155. helm/benchmark/scenarios/mmlu_pro_scenario.py +1 -1
  156. helm/benchmark/scenarios/mmmlu_scenario.py +85 -0
  157. helm/benchmark/scenarios/mtsamples_procedures_scenario.py +5 -2
  158. helm/benchmark/scenarios/mtsamples_replicate_scenario.py +3 -2
  159. helm/benchmark/scenarios/n2c2_ct_matching_scenario.py +11 -5
  160. helm/benchmark/scenarios/openai_mrcr_scenario.py +79 -0
  161. helm/benchmark/scenarios/pubmed_qa_scenario.py +6 -1
  162. helm/benchmark/scenarios/race_based_med_scenario.py +18 -8
  163. helm/benchmark/scenarios/ruler_qa_scenario_helper.py +2 -2
  164. helm/benchmark/scenarios/ruler_qa_scenarios.py +2 -2
  165. helm/benchmark/scenarios/seahelm_scenario.py +2 -2
  166. helm/benchmark/scenarios/shc_bmt_scenario.py +12 -6
  167. helm/benchmark/scenarios/shc_cdi_scenario.py +11 -6
  168. helm/benchmark/scenarios/shc_conf_scenario.py +12 -6
  169. helm/benchmark/scenarios/shc_ent_scenario.py +11 -6
  170. helm/benchmark/scenarios/shc_gip_scenario.py +13 -5
  171. helm/benchmark/scenarios/shc_privacy_scenario.py +78 -0
  172. helm/benchmark/scenarios/shc_proxy_scenario.py +76 -0
  173. helm/benchmark/scenarios/shc_ptbm_scenario.py +12 -7
  174. helm/benchmark/scenarios/shc_sei_scenario.py +12 -7
  175. helm/benchmark/scenarios/shc_sequoia_scenario.py +13 -5
  176. helm/benchmark/scenarios/starr_patient_instructions_scenario.py +15 -8
  177. helm/benchmark/scenarios/test_alghafa_scenario.py +29 -0
  178. helm/benchmark/scenarios/test_aratrust_scenario.py +21 -0
  179. helm/benchmark/scenarios/test_bluex_scenario.py +59 -0
  180. helm/benchmark/scenarios/test_exams_multilingual_scenario.py +29 -0
  181. helm/benchmark/scenarios/test_healtha_br_scenario.py +57 -0
  182. helm/benchmark/scenarios/test_infinite_bench_en_qa_scenario.py +18 -0
  183. helm/benchmark/scenarios/test_infinite_bench_en_sum_scenario.py +31 -0
  184. helm/benchmark/scenarios/truthful_qa_scenario.py +2 -1
  185. helm/benchmark/scenarios/vision_language/msr_vtt_scenario.py +75 -0
  186. helm/benchmark/server.py +2 -1
  187. helm/benchmark/slurm_jobs.py +1 -2
  188. helm/benchmark/slurm_runner.py +8 -1
  189. helm/benchmark/static/schema_arabic.yaml +228 -0
  190. helm/benchmark/static/schema_audio.yaml +60 -49
  191. helm/benchmark/static/schema_classic.yaml +0 -17
  192. helm/benchmark/static/schema_enterprise.yaml +21 -0
  193. helm/benchmark/static/schema_long_context.yaml +81 -20
  194. helm/benchmark/static/schema_medhelm.yaml +272 -213
  195. helm/benchmark/static/schema_melt.yaml +1257 -0
  196. helm/benchmark/static/schema_slphelm.yaml +162 -0
  197. helm/benchmark/static/schema_vhelm.yaml +26 -26
  198. helm/benchmark/static/schema_video.yaml +219 -0
  199. helm/benchmark/static_build/assets/index-b9779128.css +1 -0
  200. helm/benchmark/static_build/assets/index-e439d5e1.js +10 -0
  201. helm/benchmark/static_build/assets/medhelm-overview-eac29843.png +0 -0
  202. helm/benchmark/static_build/assets/{tremor-9cefc3c5.js → tremor-38a10867.js} +1 -1
  203. helm/benchmark/static_build/index.html +4 -4
  204. helm/benchmark/window_services/encoder_decoder_window_service.py +3 -3
  205. helm/benchmark/window_services/image_generation/clip_window_service.py +1 -3
  206. helm/benchmark/window_services/test_utils.py +3 -4
  207. helm/benchmark/window_services/tokenizer_service.py +7 -8
  208. helm/clients/anthropic_client.py +69 -29
  209. helm/clients/audio_language/diva_llama_client.py +4 -2
  210. helm/clients/audio_language/qwen2_5_omni_client.py +209 -0
  211. helm/clients/audio_language/qwen2_audiolm_client.py +8 -6
  212. helm/clients/audio_language/qwen_audiolm_client.py +4 -2
  213. helm/clients/audio_language/test.py +62 -0
  214. helm/clients/bedrock_client.py +3 -1
  215. helm/clients/client.py +7 -7
  216. helm/clients/grok_client.py +36 -0
  217. helm/clients/huggingface_client.py +42 -3
  218. helm/clients/huggingface_pipeline_client.py +138 -0
  219. helm/clients/image_generation/dalle_mini/model/configuration.py +1 -1
  220. helm/clients/image_generation/dalle_mini/model/modeling.py +1 -1
  221. helm/clients/image_generation/dalle_mini/model/processor.py +1 -1
  222. helm/clients/image_generation/dalle_mini/model/tokenizer.py +1 -1
  223. helm/clients/openai_client.py +102 -55
  224. helm/clients/openai_responses_client.py +176 -0
  225. helm/clients/palmyra_client.py +2 -5
  226. helm/clients/reka_client.py +2 -2
  227. helm/clients/test_huggingface_client.py +3 -3
  228. helm/clients/together_client.py +31 -6
  229. helm/clients/vertexai_client.py +17 -9
  230. helm/clients/vision_language/huggingface_vision2seq_client.py +6 -4
  231. helm/clients/vision_language/huggingface_vlm_client.py +2 -2
  232. helm/clients/vision_language/idefics_client.py +6 -2
  233. helm/clients/vision_language/paligemma_client.py +2 -2
  234. helm/clients/vision_language/qwen2_vlm_client.py +66 -53
  235. helm/clients/vision_language/qwen_vlm_client.py +7 -5
  236. helm/clients/vllm_client.py +43 -7
  237. helm/clients/vllm_granite_thinking_client.py +56 -0
  238. helm/clients/writer_client.py +102 -0
  239. helm/common/context.py +80 -0
  240. helm/common/credentials_utils.py +5 -5
  241. helm/common/critique_request.py +0 -1
  242. helm/common/general.py +9 -2
  243. helm/common/hierarchical_logger.py +104 -12
  244. helm/common/local_context.py +140 -0
  245. helm/common/object_spec.py +23 -8
  246. helm/common/remote_context.py +61 -0
  247. helm/common/request.py +8 -0
  248. helm/common/test_logging.py +94 -0
  249. helm/config/model_deployments.yaml +995 -45
  250. helm/config/model_metadata.yaml +780 -59
  251. helm/config/tokenizer_configs.yaml +224 -3
  252. helm/proxy/cli.py +4 -2
  253. helm/proxy/critique/mechanical_turk_utils.py +1 -1
  254. helm/proxy/retry.py +5 -0
  255. helm/proxy/services/server_service.py +21 -85
  256. helm/tokenizers/grok_tokenizer.py +55 -0
  257. helm/tokenizers/huggingface_tokenizer.py +1 -1
  258. helm/tokenizers/test_grok_tokenizer.py +33 -0
  259. helm/benchmark/metrics/numeracy_metrics.py +0 -72
  260. helm/benchmark/metrics/test_numeracy_metrics.py +0 -95
  261. helm/benchmark/scenarios/numeracy_scenario.py +0 -793
  262. helm/benchmark/scenarios/test_infinite_bench_sum_scenario.py +0 -46
  263. helm/benchmark/static_build/assets/index-262903c1.js +0 -10
  264. helm/benchmark/static_build/assets/index-42060d71.css +0 -1
  265. {crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/entry_points.txt +0 -0
  266. {crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/licenses/LICENSE +0 -0
  267. {crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/top_level.txt +0 -0
  268. /helm/benchmark/static_build/assets/{medhelm-overview-3ddfcd65.png → medhelm-v1-overview-3ddfcd65.png} +0 -0
{crfm_helm-0.5.5.dist-info → crfm_helm-0.5.7.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crfm-helm
3
- Version: 0.5.5
3
+ Version: 0.5.7
4
4
  Summary: Benchmark for language models
5
5
  Home-page: https://github.com/stanford-crfm/helm
6
6
  Author: Stanford CRFM
@@ -17,10 +17,11 @@ Requires-Python: >=3.9
17
17
  Description-Content-Type: text/markdown
18
18
  License-File: LICENSE
19
19
  Requires-Dist: cattrs~=22.2
20
+ Requires-Dist: colorlog~=6.9
20
21
  Requires-Dist: dacite~=1.6
21
22
  Requires-Dist: importlib-resources~=5.10
22
23
  Requires-Dist: Mako~=1.2
23
- Requires-Dist: numpy<3,~=1.26
24
+ Requires-Dist: numpy<3,>=1.26
24
25
  Requires-Dist: pandas~=2.0
25
26
  Requires-Dist: pyhocon~=0.3.59
26
27
  Requires-Dist: retrying~=1.3
@@ -29,32 +30,31 @@ Requires-Dist: tqdm~=4.64
29
30
  Requires-Dist: zstandard~=0.18.0
30
31
  Requires-Dist: sqlitedict<3.0,>=2.1.0
31
32
  Requires-Dist: bottle~=0.12.23
32
- Requires-Dist: datasets~=2.17
33
+ Requires-Dist: datasets~=3.1
33
34
  Requires-Dist: pyarrow>=11.0.0
34
35
  Requires-Dist: pyarrow-hotfix~=0.6
35
36
  Requires-Dist: nltk!=3.9.0,~=3.7
36
37
  Requires-Dist: rouge-score~=0.1.2
37
- Requires-Dist: scipy~=1.10
38
+ Requires-Dist: scipy>=1.10
38
39
  Requires-Dist: uncertainty-calibration~=0.1.4
39
- Requires-Dist: scikit-learn~=1.1
40
- Requires-Dist: transformers~=4.40
40
+ Requires-Dist: scikit-learn>=1.1
41
+ Requires-Dist: transformers<4.53.0,~=4.40
41
42
  Requires-Dist: torch<3.0.0,>=1.13.1
42
43
  Requires-Dist: torchvision<3.0.0,>=0.14.1
43
44
  Provides-Extra: proxy-server
44
45
  Requires-Dist: gunicorn>=20.1; extra == "proxy-server"
45
46
  Provides-Extra: human-evaluation
46
- Requires-Dist: scaleapi~=2.13.0; extra == "human-evaluation"
47
- Requires-Dist: surge-api~=1.1.0; extra == "human-evaluation"
47
+ Requires-Dist: scaleapi~=2.13; extra == "human-evaluation"
48
+ Requires-Dist: surge-api~=1.1; extra == "human-evaluation"
48
49
  Provides-Extra: scenarios
49
50
  Requires-Dist: gdown~=5.1; extra == "scenarios"
50
- Requires-Dist: sympy~=1.11.1; extra == "scenarios"
51
- Requires-Dist: xlrd~=2.0.1; extra == "scenarios"
51
+ Requires-Dist: xlrd~=2.0; extra == "scenarios"
52
52
  Provides-Extra: metrics
53
53
  Requires-Dist: google-api-python-client~=2.64; extra == "metrics"
54
54
  Requires-Dist: numba~=0.56; extra == "metrics"
55
- Requires-Dist: sacrebleu~=2.2.1; extra == "metrics"
56
- Requires-Dist: langdetect~=1.0.9; extra == "metrics"
57
- Requires-Dist: immutabledict~=4.2.0; extra == "metrics"
55
+ Requires-Dist: sacrebleu~=2.2; extra == "metrics"
56
+ Requires-Dist: langdetect~=1.0; extra == "metrics"
57
+ Requires-Dist: immutabledict~=4.2; extra == "metrics"
58
58
  Requires-Dist: gradio_client~=1.3; extra == "metrics"
59
59
  Provides-Extra: ranking
60
60
  Requires-Dist: pytrec_eval==0.5; extra == "ranking"
@@ -62,9 +62,9 @@ Provides-Extra: summarization
62
62
  Requires-Dist: summ-eval~=0.892; extra == "summarization"
63
63
  Requires-Dist: bert-score~=0.3; extra == "summarization"
64
64
  Provides-Extra: plots
65
- Requires-Dist: colorcet~=3.0.1; extra == "plots"
66
- Requires-Dist: matplotlib~=3.6.0; extra == "plots"
67
- Requires-Dist: seaborn~=0.11.0; extra == "plots"
65
+ Requires-Dist: colorcet~=3.0; extra == "plots"
66
+ Requires-Dist: matplotlib>=3.6.0; extra == "plots"
67
+ Requires-Dist: seaborn>=0.11.0; extra == "plots"
68
68
  Provides-Extra: decodingtrust
69
69
  Requires-Dist: fairlearn~=0.9.0; extra == "decodingtrust"
70
70
  Provides-Extra: slurm
@@ -85,29 +85,31 @@ Requires-Dist: evaluate~=0.4.1; extra == "unitxt"
85
85
  Provides-Extra: seahelm
86
86
  Requires-Dist: pythainlp==5.0.0; extra == "seahelm"
87
87
  Requires-Dist: pyonmttok==1.37.0; extra == "seahelm"
88
- Requires-Dist: sacrebleu~=2.2.1; extra == "seahelm"
88
+ Requires-Dist: sacrebleu~=2.2; extra == "seahelm"
89
89
  Requires-Dist: python-crfsuite~=0.9.11; extra == "seahelm"
90
90
  Provides-Extra: accelerate
91
91
  Requires-Dist: accelerate~=0.25; extra == "accelerate"
92
92
  Provides-Extra: aleph-alpha
93
- Requires-Dist: aleph-alpha-client~=2.14.0; extra == "aleph-alpha"
93
+ Requires-Dist: aleph-alpha-client~=2.14; extra == "aleph-alpha"
94
94
  Requires-Dist: tokenizers>=0.13.3; extra == "aleph-alpha"
95
95
  Provides-Extra: allenai
96
96
  Requires-Dist: ai2-olmo~=0.2; extra == "allenai"
97
97
  Provides-Extra: amazon
98
- Requires-Dist: boto3~=1.34.131; extra == "amazon"
99
- Requires-Dist: awscli~=1.33.13; extra == "amazon"
100
- Requires-Dist: botocore~=1.34.131; extra == "amazon"
98
+ Requires-Dist: boto3~=1.34; extra == "amazon"
99
+ Requires-Dist: awscli~=1.33; extra == "amazon"
100
+ Requires-Dist: botocore~=1.34; extra == "amazon"
101
101
  Provides-Extra: anthropic
102
- Requires-Dist: anthropic~=0.39; extra == "anthropic"
103
- Requires-Dist: websocket-client~=1.3.2; extra == "anthropic"
102
+ Requires-Dist: anthropic~=0.48; extra == "anthropic"
103
+ Requires-Dist: websocket-client~=1.3; extra == "anthropic"
104
104
  Requires-Dist: httpx<0.28.0; extra == "anthropic"
105
105
  Provides-Extra: cohere
106
106
  Requires-Dist: cohere~=5.3; extra == "cohere"
107
+ Provides-Extra: writer
108
+ Requires-Dist: writerai~=4.0; extra == "writer"
107
109
  Provides-Extra: mistral
108
110
  Requires-Dist: mistralai~=1.1; extra == "mistral"
109
111
  Provides-Extra: openai
110
- Requires-Dist: openai~=1.64; extra == "openai"
112
+ Requires-Dist: openai~=1.70; extra == "openai"
111
113
  Requires-Dist: tiktoken~=0.7; extra == "openai"
112
114
  Requires-Dist: pydantic~=2.0; extra == "openai"
113
115
  Provides-Extra: google
@@ -130,9 +132,10 @@ Requires-Dist: crfm-helm[openai]; extra == "models"
130
132
  Requires-Dist: crfm-helm[reka]; extra == "models"
131
133
  Requires-Dist: crfm-helm[together]; extra == "models"
132
134
  Requires-Dist: crfm-helm[yandex]; extra == "models"
135
+ Requires-Dist: crfm-helm[writer]; extra == "models"
133
136
  Requires-Dist: crfm-helm[ibm-enterprise-scenarios]; extra == "models"
134
137
  Provides-Extra: reka
135
- Requires-Dist: reka-api~=2.0.0; extra == "reka"
138
+ Requires-Dist: reka-api~=2.0; extra == "reka"
136
139
  Provides-Extra: vlm
137
140
  Requires-Dist: crfm-helm[openai]; extra == "vlm"
138
141
  Requires-Dist: einops~=0.7.0; extra == "vlm"
@@ -146,43 +149,47 @@ Requires-Dist: crfm-helm[reka]; extra == "vlm"
146
149
  Requires-Dist: crfm-helm[images]; extra == "vlm"
147
150
  Requires-Dist: crfm-helm[image2struct]; extra == "vlm"
148
151
  Requires-Dist: pycocoevalcap~=1.2; extra == "vlm"
149
- Requires-Dist: transformers~=4.45.2; extra == "vlm"
152
+ Requires-Dist: transformers~=4.45; extra == "vlm"
150
153
  Requires-Dist: qwen-vl-utils~=0.0.8; extra == "vlm"
151
154
  Provides-Extra: ibm-enterprise-scenarios
152
155
  Requires-Dist: openpyxl~=3.1; extra == "ibm-enterprise-scenarios"
153
156
  Provides-Extra: ibm
154
- Requires-Dist: ibm-watsonx-ai~=1.2.6; extra == "ibm"
157
+ Requires-Dist: ibm-watsonx-ai~=1.2; extra == "ibm"
155
158
  Provides-Extra: image2struct
156
159
  Requires-Dist: crfm-helm[images]; extra == "image2struct"
157
160
  Requires-Dist: latex~=0.7.0; extra == "image2struct"
158
- Requires-Dist: pdf2image~=1.16.3; extra == "image2struct"
159
- Requires-Dist: selenium~=4.17.2; extra == "image2struct"
161
+ Requires-Dist: pdf2image~=1.16; extra == "image2struct"
162
+ Requires-Dist: selenium~=4.17; extra == "image2struct"
160
163
  Requires-Dist: html2text~=2024.2.26; extra == "image2struct"
161
- Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; extra == "image2struct"
164
+ Requires-Dist: opencv-python-headless<=4.11.0.86,>=4.7.0.68; extra == "image2struct"
162
165
  Requires-Dist: lpips~=0.1.4; extra == "image2struct"
163
- Requires-Dist: imagehash~=4.3.1; extra == "image2struct"
166
+ Requires-Dist: imagehash~=4.3; extra == "image2struct"
164
167
  Provides-Extra: heim
165
168
  Requires-Dist: gdown~=5.1; extra == "heim"
166
- Requires-Dist: diffusers~=0.24.0; extra == "heim"
169
+ Requires-Dist: diffusers~=0.34.0; extra == "heim"
167
170
  Requires-Dist: icetk~=0.0.4; extra == "heim"
168
- Requires-Dist: jax~=0.4.13; extra == "heim"
169
- Requires-Dist: jaxlib~=0.4.13; extra == "heim"
171
+ Requires-Dist: jax~=0.6.2; python_version >= "3.10" and extra == "heim"
172
+ Requires-Dist: jax~=0.4.30; python_version < "3.10" and extra == "heim"
173
+ Requires-Dist: jaxlib~=0.6.2; python_version >= "3.10" and extra == "heim"
174
+ Requires-Dist: jaxlib~=0.4.30; python_version < "3.10" and extra == "heim"
170
175
  Requires-Dist: crfm-helm[openai]; extra == "heim"
171
176
  Requires-Dist: einops~=0.7.0; extra == "heim"
172
- Requires-Dist: omegaconf~=2.3.0; extra == "heim"
173
- Requires-Dist: pytorch-lightning~=2.0.5; extra == "heim"
174
- Requires-Dist: flax~=0.6.11; extra == "heim"
175
- Requires-Dist: ftfy~=6.1.1; extra == "heim"
176
- Requires-Dist: Unidecode~=1.3.6; extra == "heim"
177
+ Requires-Dist: omegaconf~=2.3; extra == "heim"
178
+ Requires-Dist: pytorch-lightning~=2.0; extra == "heim"
179
+ Requires-Dist: flax~=0.10.7; python_version >= "3.10" and extra == "heim"
180
+ Requires-Dist: flax~=0.8.5; python_version < "3.10" and extra == "heim"
181
+ Requires-Dist: ftfy~=6.1; extra == "heim"
182
+ Requires-Dist: Unidecode~=1.3; extra == "heim"
177
183
  Requires-Dist: wandb~=0.16; extra == "heim"
178
- Requires-Dist: google-cloud-translate~=3.11.2; extra == "heim"
179
- Requires-Dist: autokeras~=1.0.20; extra == "heim"
180
- Requires-Dist: clip-anytorch~=2.5.0; extra == "heim"
184
+ Requires-Dist: google-cloud-translate~=3.11; extra == "heim"
185
+ Requires-Dist: autokeras~=1.0; extra == "heim"
186
+ Requires-Dist: clip-anytorch~=2.5; extra == "heim"
181
187
  Requires-Dist: google-cloud-storage~=2.9; extra == "heim"
182
188
  Requires-Dist: lpips~=0.1.4; extra == "heim"
183
- Requires-Dist: multilingual-clip~=1.0.10; extra == "heim"
184
- Requires-Dist: NudeNet~=2.0.9; extra == "heim"
185
- Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; extra == "heim"
189
+ Requires-Dist: multilingual-clip~=1.0; extra == "heim"
190
+ Requires-Dist: NudeNet~=2.0; extra == "heim"
191
+ Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; python_version >= "3.10" and extra == "heim"
192
+ Requires-Dist: opencv-python-headless<=4.11.0.86,>=4.7.0.68; python_version < "3.10" and extra == "heim"
186
193
  Requires-Dist: pytorch-fid~=0.3.0; extra == "heim"
187
194
  Requires-Dist: tensorflow~=2.11; extra == "heim"
188
195
  Requires-Dist: timm~=0.6.12; extra == "heim"
@@ -191,10 +198,14 @@ Requires-Dist: torchmetrics~=0.11.1; extra == "heim"
191
198
  Requires-Dist: scikit-image!=0.23.*,==0.*,>=0.22; extra == "heim"
192
199
  Requires-Dist: crfm-helm[images]; extra == "heim"
193
200
  Provides-Extra: medhelm
194
- Requires-Dist: crfm-helm[summarization]; extra == "medhelm"
195
- Requires-Dist: python-docx~=1.1.2; extra == "medhelm"
196
- Requires-Dist: langchain~=0.3.9; extra == "medhelm"
197
- Requires-Dist: lxml~=5.3.0; extra == "medhelm"
201
+ Requires-Dist: accelerate~=0.25; extra == "medhelm"
202
+ Requires-Dist: crfm-helm[openai]; extra == "medhelm"
203
+ Requires-Dist: crfm-helm[yandex]; extra == "medhelm"
204
+ Requires-Dist: bert_score~=0.3.13; extra == "medhelm"
205
+ Requires-Dist: lxml~=5.3; extra == "medhelm"
206
+ Requires-Dist: openpyxl~=3.1; extra == "medhelm"
207
+ Requires-Dist: python-docx~=1.1; extra == "medhelm"
208
+ Requires-Dist: transformers<4.50,~=4.45; extra == "medhelm"
198
209
  Provides-Extra: audiolm
199
210
  Requires-Dist: crfm-helm[openai]; extra == "audiolm"
200
211
  Requires-Dist: crfm-helm[google]; extra == "audiolm"
@@ -204,14 +215,21 @@ Requires-Dist: soundfile~=0.12; extra == "audiolm"
204
215
  Requires-Dist: librosa~=0.10; extra == "audiolm"
205
216
  Requires-Dist: einops~=0.7.0; extra == "audiolm"
206
217
  Requires-Dist: openai-whisper==20240930; extra == "audiolm"
207
- Requires-Dist: transformers~=4.45.1; extra == "audiolm"
218
+ Requires-Dist: transformers~=4.48; extra == "audiolm"
208
219
  Requires-Dist: transformers_stream_generator~=0.0.4; extra == "audiolm"
220
+ Requires-Dist: av~=14.3; extra == "audiolm"
209
221
  Requires-Dist: scipy~=1.10; extra == "audiolm"
210
222
  Requires-Dist: torchvision<3.0.0,>=0.14.1; extra == "audiolm"
223
+ Requires-Dist: flash-attn~=2.7; extra == "audiolm"
211
224
  Requires-Dist: pycocoevalcap~=1.2; extra == "audiolm"
212
225
  Requires-Dist: jiwer~=3.0; extra == "audiolm"
213
226
  Requires-Dist: rapidfuzz~=3.10; extra == "audiolm"
214
227
  Requires-Dist: jieba~=0.42.1; extra == "audiolm"
228
+ Provides-Extra: codeinsights
229
+ Requires-Dist: clang~=20.1; extra == "codeinsights"
230
+ Requires-Dist: Levenshtein~=0.27; extra == "codeinsights"
231
+ Provides-Extra: lmkt
232
+ Requires-Dist: sentence_transformers~=4.1; extra == "lmkt"
215
233
  Provides-Extra: all
216
234
  Requires-Dist: crfm-helm[proxy-server]; extra == "all"
217
235
  Requires-Dist: crfm-helm[human-evaluation]; extra == "all"
@@ -226,12 +244,14 @@ Requires-Dist: crfm-helm[models]; extra == "all"
226
244
  Requires-Dist: crfm-helm[mongo]; extra == "all"
227
245
  Requires-Dist: crfm-helm[heim]; extra == "all"
228
246
  Requires-Dist: crfm-helm[vlm]; extra == "all"
229
- Requires-Dist: crfm-helm[audiolm]; extra == "all"
247
+ Requires-Dist: crfm-helm[codeinsights]; extra == "all"
248
+ Requires-Dist: crfm-helm[lmkt]; extra == "all"
230
249
  Provides-Extra: dev
231
250
  Requires-Dist: pytest~=7.2.0; extra == "dev"
251
+ Requires-Dist: xdoctest~=1.2.0; extra == "dev"
232
252
  Requires-Dist: pre-commit~=2.20.0; extra == "dev"
233
253
  Requires-Dist: black==24.3.0; extra == "dev"
234
- Requires-Dist: mypy==1.5.1; extra == "dev"
254
+ Requires-Dist: mypy==1.16.0; extra == "dev"
235
255
  Requires-Dist: flake8==5.0.4; extra == "dev"
236
256
  Dynamic: license-file
237
257
 
@@ -318,8 +338,9 @@ The HELM framework was used in the following papers for evaluating models.
318
338
  - **Holistic Evaluation of Text-To-Image Models (HEIM)** - [paper](https://arxiv.org/abs/2311.04287), [leaderboard](https://crfm.stanford.edu/helm/heim/latest/), [documentation](https://crfm-helm.readthedocs.io/en/latest/heim/)
319
339
  - **Image2Struct: Benchmarking Structure Extraction for Vision-Language Models** - [paper](https://arxiv.org/abs/2410.22456)
320
340
  - **Enterprise Benchmarks for Large Language Model Evaluation** - [paper](https://arxiv.org/abs/2410.12857), [documentation](https://crfm-helm.readthedocs.io/en/latest/enterprise_benchmark/)
321
- - **The Mighty ToRR: A Benchmark for Table Reasoning and Robustness** - [paper](https://arxiv.org/abs/2502.19412)
341
+ - **The Mighty ToRR: A Benchmark for Table Reasoning and Robustness** - [paper](https://arxiv.org/abs/2502.19412), [leaderboard](https://crfm.stanford.edu/helm/torr/latest/)
322
342
  - **Reliable and Efficient Amortized Model-based Evaluation** - [paper](https://arxiv.org/abs/2503.13335), [documentation](https://crfm-helm.readthedocs.io/en/latest/reeval/)
343
+ - **MedHELM** - paper in progress, [leaderboard](https://crfm.stanford.edu/helm/medhelm/latest/), [documentation](https://crfm-helm.readthedocs.io/en/latest/reeval/)
323
344
 
324
345
  The HELM framework can be used to reproduce the published model evaluation results from these papers. To get started, refer to the documentation links above for the corresponding paper, or the [main Reproducing Leaderboards documentation](https://crfm-helm.readthedocs.io/en/latest/reproducing_leaderboards/).
325
346