crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crfm-helm might be problematic. Click here for more details.

Files changed (1033) hide show
  1. crfm_helm-0.5.10.dist-info/METADATA +369 -0
  2. crfm_helm-0.5.10.dist-info/RECORD +1008 -0
  3. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.10.dist-info}/WHEEL +1 -1
  4. helm/benchmark/adaptation/adapter_spec.py +80 -29
  5. helm/benchmark/adaptation/adapters/adapter.py +2 -2
  6. helm/benchmark/adaptation/adapters/adapter_factory.py +39 -28
  7. helm/benchmark/adaptation/adapters/binary_ranking_adapter.py +1 -1
  8. helm/benchmark/adaptation/adapters/chat_adapter.py +49 -0
  9. helm/benchmark/adaptation/adapters/ehr_instruction_adapter.py +108 -0
  10. helm/benchmark/adaptation/adapters/generation_adapter.py +2 -1
  11. helm/benchmark/adaptation/adapters/in_context_learning_adapter.py +24 -8
  12. helm/benchmark/adaptation/adapters/language_modeling_adapter.py +3 -4
  13. helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py +4 -2
  14. helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py +2 -1
  15. helm/benchmark/adaptation/adapters/multimodal/multimodal_prompt.py +7 -0
  16. helm/benchmark/adaptation/adapters/multimodal/multiple_choice_joint_multimodal_adapter.py +112 -0
  17. helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py +6 -3
  18. helm/benchmark/adaptation/adapters/multimodal/test_multimodal_prompt.py +3 -1
  19. helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py +1 -1
  20. helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py +18 -8
  21. helm/benchmark/adaptation/adapters/multiple_choice_joint_chain_of_thought_adapter.py +87 -0
  22. helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py +1 -1
  23. helm/benchmark/adaptation/adapters/test_adapter.py +5 -4
  24. helm/benchmark/adaptation/adapters/test_generation_adapter.py +46 -22
  25. helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py +17 -29
  26. helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py +138 -16
  27. helm/benchmark/adaptation/common_adapter_specs.py +443 -0
  28. helm/benchmark/adaptation/prompt.py +1 -1
  29. helm/benchmark/adaptation/request_state.py +6 -1
  30. helm/benchmark/adaptation/scenario_state.py +6 -2
  31. helm/benchmark/annotation/aci_bench_annotator.py +84 -0
  32. helm/benchmark/annotation/air_bench_annotator.py +79 -0
  33. helm/benchmark/annotation/alrage_annotator.py +90 -0
  34. helm/benchmark/annotation/annotator.py +48 -0
  35. helm/benchmark/annotation/annotator_factory.py +50 -0
  36. helm/benchmark/annotation/anthropic_red_team_annotator.py +57 -0
  37. helm/benchmark/annotation/autobencher_capabilities_annotator.py +107 -0
  38. helm/benchmark/annotation/autobencher_safety_annotator.py +98 -0
  39. helm/benchmark/annotation/bigcodebench_annotator.py +108 -0
  40. helm/benchmark/annotation/bird_sql_annotator.py +58 -0
  41. helm/benchmark/annotation/call_center_annotator.py +258 -0
  42. helm/benchmark/annotation/chw_care_plan_annotator.py +82 -0
  43. helm/benchmark/annotation/czech_bank_qa_annotator.py +78 -0
  44. helm/benchmark/annotation/dischargeme_annotator.py +96 -0
  45. helm/benchmark/annotation/ehr_sql_annotator.py +87 -0
  46. helm/benchmark/annotation/financebench_annotator.py +79 -0
  47. helm/benchmark/annotation/harm_bench_annotator.py +55 -0
  48. helm/benchmark/annotation/helpdesk_call_summarization_annotator.py +131 -0
  49. helm/benchmark/annotation/image2struct/image_compiler_annotator.py +93 -0
  50. helm/benchmark/annotation/image2struct/latex_compiler_annotator.py +59 -0
  51. helm/benchmark/annotation/image2struct/lilypond_compiler_annotator.py +86 -0
  52. helm/benchmark/annotation/image2struct/webpage_compiler_annotator.py +132 -0
  53. helm/benchmark/annotation/live_qa_annotator.py +76 -0
  54. helm/benchmark/annotation/med_dialog_annotator.py +88 -0
  55. helm/benchmark/annotation/medalign_annotator.py +89 -0
  56. helm/benchmark/annotation/medi_qa_annotator.py +87 -0
  57. helm/benchmark/annotation/medication_qa_annotator.py +86 -0
  58. helm/benchmark/annotation/mental_health_annotator.py +87 -0
  59. helm/benchmark/annotation/mimic_bhc_annotator.py +89 -0
  60. helm/benchmark/annotation/mimic_rrs_annotator.py +89 -0
  61. helm/benchmark/annotation/model_as_judge.py +309 -0
  62. helm/benchmark/annotation/mtsamples_procedures_annotator.py +87 -0
  63. helm/benchmark/annotation/mtsamples_replicate_annotator.py +90 -0
  64. helm/benchmark/annotation/omni_math/gpt_evaluation_template.txt +152 -0
  65. helm/benchmark/annotation/omni_math/gpt_evaluation_zero_shot_template.txt +36 -0
  66. helm/benchmark/annotation/omni_math_annotator.py +131 -0
  67. helm/benchmark/annotation/simple_safety_tests_annotator.py +50 -0
  68. helm/benchmark/annotation/spider_annotator.py +18 -0
  69. helm/benchmark/annotation/starr_patient_instructions_annotator.py +87 -0
  70. helm/benchmark/annotation/test_annotator_factory.py +26 -0
  71. helm/benchmark/annotation/test_dummy_annotator.py +44 -0
  72. helm/benchmark/annotation/wildbench/eval_template.pairwise.v2.md +75 -0
  73. helm/benchmark/annotation/wildbench/eval_template.score.v2.md +66 -0
  74. helm/benchmark/annotation/wildbench_annotator.py +119 -0
  75. helm/benchmark/annotation/xstest_annotator.py +100 -0
  76. helm/benchmark/annotation_executor.py +144 -0
  77. helm/benchmark/augmentations/cleva_perturbation.py +9 -8
  78. helm/benchmark/augmentations/contraction_expansion_perturbation.py +2 -2
  79. helm/benchmark/augmentations/contrast_sets_perturbation.py +2 -2
  80. helm/benchmark/augmentations/data_augmenter.py +0 -2
  81. helm/benchmark/augmentations/dialect_perturbation.py +4 -5
  82. helm/benchmark/augmentations/extra_space_perturbation.py +2 -2
  83. helm/benchmark/augmentations/filler_words_perturbation.py +2 -2
  84. helm/benchmark/augmentations/gender_perturbation.py +3 -3
  85. helm/benchmark/augmentations/lowercase_perturbation.py +2 -2
  86. helm/benchmark/augmentations/mild_mix_perturbation.py +6 -6
  87. helm/benchmark/augmentations/misspelling_perturbation.py +2 -2
  88. helm/benchmark/augmentations/person_name_perturbation.py +4 -5
  89. helm/benchmark/augmentations/perturbation.py +26 -4
  90. helm/benchmark/augmentations/perturbation_description.py +1 -1
  91. helm/benchmark/augmentations/space_perturbation.py +2 -2
  92. helm/benchmark/augmentations/suffix_perturbation.py +29 -0
  93. helm/benchmark/augmentations/synonym_perturbation.py +4 -3
  94. helm/benchmark/augmentations/test_perturbation.py +56 -19
  95. helm/benchmark/augmentations/translate_perturbation.py +31 -0
  96. helm/benchmark/augmentations/typos_perturbation.py +2 -2
  97. helm/benchmark/config_registry.py +7 -1
  98. helm/benchmark/data_preprocessor.py +2 -2
  99. helm/benchmark/executor.py +54 -25
  100. helm/benchmark/huggingface_registration.py +28 -10
  101. helm/benchmark/metrics/air_bench_metrics.py +3212 -0
  102. helm/benchmark/metrics/alrage_metric.py +35 -0
  103. helm/benchmark/metrics/annotation_metrics.py +108 -0
  104. helm/benchmark/metrics/basic_metrics.py +437 -667
  105. helm/benchmark/metrics/bbq_metrics.py +17 -6
  106. helm/benchmark/metrics/bias_metrics.py +18 -9
  107. helm/benchmark/metrics/bias_word_lists.py +1 -1
  108. helm/benchmark/metrics/bigcodebench_metrics.py +25 -0
  109. helm/benchmark/metrics/bird_sql_metrics.py +28 -0
  110. helm/benchmark/metrics/classification_metrics.py +107 -22
  111. helm/benchmark/metrics/cleva_accuracy_metrics.py +8 -5
  112. helm/benchmark/metrics/cleva_harms_metrics.py +12 -11
  113. helm/benchmark/metrics/code_metrics.py +5 -5
  114. helm/benchmark/metrics/code_metrics_helper.py +11 -3
  115. helm/benchmark/metrics/codeinsights_code_efficiency_metrics.py +186 -0
  116. helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py +477 -0
  117. helm/benchmark/metrics/codeinsights_correct_code_metrics.py +366 -0
  118. helm/benchmark/metrics/codeinsights_edge_case_metrics.py +92 -0
  119. helm/benchmark/metrics/codeinsights_metric_specs.py +51 -0
  120. helm/benchmark/metrics/comet_metric.py +125 -0
  121. helm/benchmark/metrics/common_metric_specs.py +174 -0
  122. helm/benchmark/metrics/conv_fin_qa_calc_metrics.py +83 -0
  123. helm/benchmark/metrics/copyright_metrics.py +5 -5
  124. helm/benchmark/metrics/czech_bank_qa_metrics.py +29 -0
  125. helm/benchmark/metrics/decodingtrust_fairness_metrics.py +72 -0
  126. helm/benchmark/metrics/decodingtrust_ood_knowledge_metrics.py +66 -0
  127. helm/benchmark/metrics/decodingtrust_privacy_metrics.py +101 -0
  128. helm/benchmark/metrics/decodingtrust_stereotype_bias_metrics.py +202 -0
  129. helm/benchmark/metrics/disinformation_metrics.py +8 -114
  130. helm/benchmark/metrics/dry_run_metrics.py +35 -6
  131. helm/benchmark/metrics/efficiency_metrics.py +287 -0
  132. helm/benchmark/metrics/ehr_sql_metrics.py +159 -0
  133. helm/benchmark/metrics/evaluate_instances_metric.py +59 -0
  134. helm/benchmark/metrics/evaluate_reference_metrics.py +831 -0
  135. helm/benchmark/metrics/fin_qa_metrics.py +60 -0
  136. helm/benchmark/metrics/fin_qa_metrics_helper.py +398 -0
  137. helm/benchmark/metrics/gpqa_chain_of_thought_metric.py +115 -0
  138. helm/benchmark/metrics/gpt4_audio_critique_metrics.py +167 -0
  139. helm/benchmark/metrics/gpt4_audio_refusal_metrics.py +145 -0
  140. helm/benchmark/metrics/gpt4v_originality_critique_metrics.py +126 -0
  141. helm/benchmark/metrics/helpdesk_call_summarization_metrics.py +48 -0
  142. helm/benchmark/metrics/ifeval/instructions.py +1574 -0
  143. helm/benchmark/metrics/ifeval/instructions_registry.py +182 -0
  144. helm/benchmark/metrics/ifeval/instructions_registry.pyi +3 -0
  145. helm/benchmark/metrics/ifeval/instructions_util.py +153 -0
  146. helm/benchmark/metrics/ifeval_metrics.py +67 -0
  147. helm/benchmark/metrics/image_generation/aesthetics_metrics.py +54 -0
  148. helm/benchmark/metrics/image_generation/aesthetics_scorer.py +66 -0
  149. helm/benchmark/metrics/image_generation/clip_score_metrics.py +84 -0
  150. helm/benchmark/metrics/image_generation/denoised_runtime_metric.py +42 -0
  151. helm/benchmark/metrics/image_generation/detection_metrics.py +57 -0
  152. helm/benchmark/metrics/image_generation/detectors/base_detector.py +8 -0
  153. helm/benchmark/metrics/image_generation/detectors/vitdet.py +178 -0
  154. helm/benchmark/metrics/image_generation/efficiency_metrics.py +41 -0
  155. helm/benchmark/metrics/image_generation/fidelity_metrics.py +168 -0
  156. helm/benchmark/metrics/image_generation/fractal_dimension/__init__.py +0 -0
  157. helm/benchmark/metrics/image_generation/fractal_dimension/fractal_dimension_util.py +63 -0
  158. helm/benchmark/metrics/image_generation/fractal_dimension/test_fractal_dimension_util.py +33 -0
  159. helm/benchmark/metrics/image_generation/fractal_dimension_metric.py +50 -0
  160. helm/benchmark/metrics/image_generation/gender_metrics.py +58 -0
  161. helm/benchmark/metrics/image_generation/image_critique_metrics.py +284 -0
  162. helm/benchmark/metrics/image_generation/lpips_metrics.py +82 -0
  163. helm/benchmark/metrics/image_generation/multi_scale_ssim_metrics.py +82 -0
  164. helm/benchmark/metrics/image_generation/nsfw_detector.py +96 -0
  165. helm/benchmark/metrics/image_generation/nsfw_metrics.py +103 -0
  166. helm/benchmark/metrics/image_generation/nudity_metrics.py +38 -0
  167. helm/benchmark/metrics/image_generation/photorealism_critique_metrics.py +153 -0
  168. helm/benchmark/metrics/image_generation/psnr_metrics.py +78 -0
  169. helm/benchmark/metrics/image_generation/q16/__init__.py +0 -0
  170. helm/benchmark/metrics/image_generation/q16/q16_toxicity_detector.py +90 -0
  171. helm/benchmark/metrics/image_generation/q16/test_q16.py +20 -0
  172. helm/benchmark/metrics/image_generation/q16_toxicity_metrics.py +48 -0
  173. helm/benchmark/metrics/image_generation/skin_tone_metrics.py +164 -0
  174. helm/benchmark/metrics/image_generation/uiqi_metrics.py +92 -0
  175. helm/benchmark/metrics/image_generation/watermark/__init__.py +0 -0
  176. helm/benchmark/metrics/image_generation/watermark/test_watermark_detector.py +16 -0
  177. helm/benchmark/metrics/image_generation/watermark/watermark_detector.py +87 -0
  178. helm/benchmark/metrics/image_generation/watermark_metrics.py +48 -0
  179. helm/benchmark/metrics/instruction_following_critique_metrics.py +48 -5
  180. helm/benchmark/metrics/kpi_edgar_metrics.py +142 -0
  181. helm/benchmark/metrics/language_modeling_metrics.py +111 -0
  182. helm/benchmark/metrics/live_qa_metrics.py +35 -0
  183. helm/benchmark/metrics/llm_jury_metrics.py +58 -0
  184. helm/benchmark/metrics/lmkt_metric_specs.py +12 -0
  185. helm/benchmark/metrics/lmkt_metrics.py +47 -0
  186. helm/benchmark/metrics/machine_translation_metrics.py +89 -0
  187. helm/benchmark/metrics/medcalc_bench_metrics.py +137 -0
  188. helm/benchmark/metrics/medec_metrics.py +124 -0
  189. helm/benchmark/metrics/melt_bias_metric.py +234 -0
  190. helm/benchmark/metrics/melt_bias_word_lists.py +1367 -0
  191. helm/benchmark/metrics/melt_metric_specs.py +43 -0
  192. helm/benchmark/metrics/melt_toxicity_metric.py +107 -0
  193. helm/benchmark/metrics/metric.py +121 -175
  194. helm/benchmark/metrics/metric_name.py +0 -1
  195. helm/benchmark/metrics/metric_service.py +23 -7
  196. helm/benchmark/metrics/mimiciv_billing_code_metrics.py +127 -0
  197. helm/benchmark/metrics/nltk_helper.py +32 -0
  198. helm/benchmark/metrics/omni_math_metrics.py +44 -0
  199. helm/benchmark/metrics/openai_mrcr_metrics.py +52 -0
  200. helm/benchmark/metrics/output_processing_metric.py +60 -0
  201. helm/benchmark/metrics/output_processors.py +15 -0
  202. helm/benchmark/metrics/paraphrase_generation_metrics.py +5 -6
  203. helm/benchmark/metrics/prometheus_vision_critique_metrics.py +185 -0
  204. helm/benchmark/metrics/ranking_metrics.py +5 -5
  205. helm/benchmark/metrics/reference_metric.py +148 -0
  206. helm/benchmark/metrics/reka_vibe_critique_metrics.py +158 -0
  207. helm/benchmark/metrics/ruler_qa_metrics.py +34 -0
  208. helm/benchmark/metrics/safety_metrics.py +91 -0
  209. helm/benchmark/metrics/seahelm_metrics.py +201 -0
  210. helm/benchmark/metrics/seahelm_metrics_specs.py +10 -0
  211. helm/benchmark/metrics/spider_metrics.py +7 -0
  212. helm/benchmark/metrics/statistic.py +1 -1
  213. helm/benchmark/metrics/summac/model_summac.py +8 -11
  214. helm/benchmark/metrics/summarization_critique_metrics.py +4 -4
  215. helm/benchmark/metrics/summarization_metrics.py +150 -11
  216. helm/benchmark/metrics/test_bias_metrics.py +5 -1
  217. helm/benchmark/metrics/test_classification_metrics.py +145 -70
  218. helm/benchmark/metrics/test_disinformation_metrics.py +78 -0
  219. helm/benchmark/metrics/{test_basic_metrics.py → test_evaluate_reference_metrics.py} +20 -1
  220. helm/benchmark/metrics/test_metric.py +3 -3
  221. helm/benchmark/metrics/test_statistic.py +2 -2
  222. helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py +1 -1
  223. helm/benchmark/metrics/tokens/auto_token_cost_estimator.py +6 -6
  224. helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py +1 -1
  225. helm/benchmark/metrics/tokens/free_token_cost_estimator.py +1 -1
  226. helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py +11 -3
  227. helm/benchmark/metrics/tokens/openai_token_cost_estimator.py +1 -1
  228. helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py +3 -3
  229. helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py +7 -7
  230. helm/benchmark/metrics/toxicity_metrics.py +37 -7
  231. helm/benchmark/metrics/toxicity_utils.py +23 -0
  232. helm/benchmark/metrics/ultra_suite_asr_classification_metrics.py +52 -0
  233. helm/benchmark/metrics/unitxt_metrics.py +107 -0
  234. helm/benchmark/metrics/vision_language/__init__.py +0 -0
  235. helm/benchmark/metrics/vision_language/emd_utils.py +347 -0
  236. helm/benchmark/metrics/vision_language/image_metrics.py +537 -0
  237. helm/benchmark/metrics/vision_language/image_utils.py +100 -0
  238. helm/benchmark/metrics/wildbench_metrics.py +54 -0
  239. helm/benchmark/model_deployment_registry.py +69 -5
  240. helm/benchmark/model_metadata_registry.py +58 -2
  241. helm/benchmark/multi_gpu_runner.py +133 -0
  242. helm/benchmark/presentation/contamination.py +3 -3
  243. helm/benchmark/presentation/create_plots.py +51 -20
  244. helm/benchmark/presentation/run_display.py +51 -12
  245. helm/benchmark/presentation/run_entry.py +2 -2
  246. helm/benchmark/presentation/schema.py +83 -66
  247. helm/benchmark/presentation/summarize.py +483 -388
  248. helm/benchmark/presentation/table.py +8 -8
  249. helm/benchmark/presentation/taxonomy_info.py +20 -0
  250. helm/benchmark/presentation/test_contamination.py +2 -2
  251. helm/benchmark/presentation/test_create_plots.py +4 -1
  252. helm/benchmark/presentation/test_run_entry.py +2 -2
  253. helm/benchmark/presentation/test_schema.py +11 -0
  254. helm/benchmark/presentation/test_summarize.py +148 -6
  255. helm/benchmark/presentation/torr_robustness_summarizer.py +178 -0
  256. helm/benchmark/reeval_run.py +202 -0
  257. helm/benchmark/reeval_runner.py +355 -0
  258. helm/benchmark/run.py +151 -87
  259. helm/benchmark/run_expander.py +418 -33
  260. helm/benchmark/run_spec.py +93 -0
  261. helm/benchmark/run_spec_factory.py +180 -0
  262. helm/benchmark/run_specs/__init__.py +0 -0
  263. helm/benchmark/run_specs/air_bench_run_specs.py +58 -0
  264. helm/benchmark/run_specs/arabic_run_specs.py +197 -0
  265. helm/benchmark/run_specs/audio_run_specs.py +657 -0
  266. helm/benchmark/run_specs/bluex_run_specs.py +40 -0
  267. helm/benchmark/run_specs/call_center_run_specs.py +201 -0
  268. helm/benchmark/run_specs/capabilities_run_specs.py +308 -0
  269. helm/benchmark/run_specs/classic_run_specs.py +1393 -0
  270. helm/benchmark/run_specs/cleva_run_specs.py +277 -0
  271. helm/benchmark/run_specs/codeinsights_run_specs.py +192 -0
  272. helm/benchmark/run_specs/decodingtrust_run_specs.py +316 -0
  273. helm/benchmark/run_specs/enem_challenge_specs.py +31 -0
  274. helm/benchmark/run_specs/enterprise_run_specs.py +280 -0
  275. helm/benchmark/run_specs/experimental_run_specs.py +224 -0
  276. helm/benchmark/run_specs/finance_run_specs.py +114 -0
  277. helm/benchmark/run_specs/healthqa_br_run_specs.py +40 -0
  278. helm/benchmark/run_specs/heim_run_specs.py +625 -0
  279. helm/benchmark/run_specs/imdb_ptbr_run_specs.py +30 -0
  280. helm/benchmark/run_specs/instruction_following_run_specs.py +129 -0
  281. helm/benchmark/run_specs/lite_run_specs.py +307 -0
  282. helm/benchmark/run_specs/lmkt_run_specs.py +144 -0
  283. helm/benchmark/run_specs/long_context_run_specs.py +188 -0
  284. helm/benchmark/run_specs/medhelm/__init__.py +0 -0
  285. helm/benchmark/run_specs/medhelm/benchmark_config.py +219 -0
  286. helm/benchmark/run_specs/medhelm_run_specs.py +1570 -0
  287. helm/benchmark/run_specs/melt_run_specs.py +783 -0
  288. helm/benchmark/run_specs/mmlu_clinical_afr_run_specs.py +49 -0
  289. helm/benchmark/run_specs/multilingual_run_specs.py +50 -0
  290. helm/benchmark/run_specs/oab_exams_specs.py +32 -0
  291. helm/benchmark/run_specs/safety_run_specs.py +191 -0
  292. helm/benchmark/run_specs/seahelm_run_specs.py +652 -0
  293. helm/benchmark/run_specs/simple_run_specs.py +104 -0
  294. helm/benchmark/run_specs/speech_disorder_audio_run_specs.py +167 -0
  295. helm/benchmark/run_specs/sql_run_specs.py +54 -0
  296. helm/benchmark/run_specs/tweetsentbr_run_specs.py +32 -0
  297. helm/benchmark/run_specs/unitxt_run_specs.py +51 -0
  298. helm/benchmark/run_specs/vlm_run_specs.py +1057 -0
  299. helm/benchmark/run_specs/winogrande_afr_run_specs.py +47 -0
  300. helm/benchmark/runner.py +63 -62
  301. helm/benchmark/runner_config_registry.py +21 -0
  302. helm/benchmark/scenarios/aci_bench_scenario.py +149 -0
  303. helm/benchmark/scenarios/air_bench_scenario.py +76 -0
  304. helm/benchmark/scenarios/alghafa_scenario.py +126 -0
  305. helm/benchmark/scenarios/alrage_scenario.py +54 -0
  306. helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py +27 -3
  307. helm/benchmark/scenarios/anthropic_red_team_scenario.py +82 -0
  308. helm/benchmark/scenarios/arabic_exams_scenario.py +114 -0
  309. helm/benchmark/scenarios/arabic_mmlu_scenario.py +82 -0
  310. helm/benchmark/scenarios/aratrust_scenario.py +95 -0
  311. helm/benchmark/scenarios/audio_language/__init__.py +0 -0
  312. helm/benchmark/scenarios/audio_language/air_bench_chat_scenario.py +130 -0
  313. helm/benchmark/scenarios/audio_language/air_bench_foundation_scenario.py +154 -0
  314. helm/benchmark/scenarios/audio_language/ami_scenario.py +96 -0
  315. helm/benchmark/scenarios/audio_language/audio_mnist_scenario.py +62 -0
  316. helm/benchmark/scenarios/audio_language/audio_pairs_scenario.py +62 -0
  317. helm/benchmark/scenarios/audio_language/audiocaps_scenario.py +59 -0
  318. helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py +152 -0
  319. helm/benchmark/scenarios/audio_language/common_voice_15_scenario.py +99 -0
  320. helm/benchmark/scenarios/audio_language/corebench_scenario.py +77 -0
  321. helm/benchmark/scenarios/audio_language/covost2_scenario.py +163 -0
  322. helm/benchmark/scenarios/audio_language/fleurs_fairness_scenario.py +83 -0
  323. helm/benchmark/scenarios/audio_language/fleurs_scenario.py +312 -0
  324. helm/benchmark/scenarios/audio_language/iemocap_audio_scenario.py +83 -0
  325. helm/benchmark/scenarios/audio_language/librispeech_fairness_scenario.py +96 -0
  326. helm/benchmark/scenarios/audio_language/librispeech_scenario.py +80 -0
  327. helm/benchmark/scenarios/audio_language/meld_audio_scenario.py +113 -0
  328. helm/benchmark/scenarios/audio_language/multilingual_librispeech_scenario.py +80 -0
  329. helm/benchmark/scenarios/audio_language/mustard_scenario.py +142 -0
  330. helm/benchmark/scenarios/audio_language/mutox_scenario.py +254 -0
  331. helm/benchmark/scenarios/audio_language/parade_scenario.py +97 -0
  332. helm/benchmark/scenarios/audio_language/speech_robust_bench_scenario.py +124 -0
  333. helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification_scenario.py +74 -0
  334. helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py +70 -0
  335. helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py +79 -0
  336. helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py +78 -0
  337. helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py +78 -0
  338. helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py +83 -0
  339. helm/benchmark/scenarios/audio_language/voice_jailbreak_attacks_scenario.py +87 -0
  340. helm/benchmark/scenarios/audio_language/voxceleb2_scenario.py +105 -0
  341. helm/benchmark/scenarios/autobencher_capabilities_scenario.py +68 -0
  342. helm/benchmark/scenarios/autobencher_safety_scenario.py +51 -0
  343. helm/benchmark/scenarios/babi_qa_scenario.py +16 -1
  344. helm/benchmark/scenarios/banking77_scenario.py +77 -0
  345. helm/benchmark/scenarios/bbq_scenario.py +17 -2
  346. helm/benchmark/scenarios/best_chatgpt_prompts.yaml +473 -0
  347. helm/benchmark/scenarios/big_bench_scenario.py +11 -1
  348. helm/benchmark/scenarios/bigcodebench_scenario.py +58 -0
  349. helm/benchmark/scenarios/bird_sql_scenario.py +112 -0
  350. helm/benchmark/scenarios/bird_sql_scenario_helper.py +118 -0
  351. helm/benchmark/scenarios/blimp_scenario.py +1 -1
  352. helm/benchmark/scenarios/bluex_scenario.py +70 -0
  353. helm/benchmark/scenarios/bold_scenario.py +18 -3
  354. helm/benchmark/scenarios/boolq_scenario.py +21 -1
  355. helm/benchmark/scenarios/call_center_scenario.py +84 -0
  356. helm/benchmark/scenarios/casehold_scenario.py +79 -0
  357. helm/benchmark/scenarios/chw_care_plan_scenario.py +129 -0
  358. helm/benchmark/scenarios/ci_mcqa_scenario.py +80 -0
  359. helm/benchmark/scenarios/civil_comments_scenario.py +14 -1
  360. helm/benchmark/scenarios/clear_scenario.py +180 -0
  361. helm/benchmark/scenarios/cleva_scenario.py +482 -3
  362. helm/benchmark/scenarios/code_scenario.py +46 -4
  363. helm/benchmark/scenarios/codeinsights_code_efficiency_scenario.py +197 -0
  364. helm/benchmark/scenarios/codeinsights_correct_code_scenario.py +78 -0
  365. helm/benchmark/scenarios/codeinsights_edge_case_scenario.py +192 -0
  366. helm/benchmark/scenarios/codeinsights_student_coding_scenario.py +162 -0
  367. helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py +188 -0
  368. helm/benchmark/scenarios/commonsense_scenario.py +33 -1
  369. helm/benchmark/scenarios/compositional_instructions.yaml +70 -0
  370. helm/benchmark/scenarios/conv_fin_qa_calc_scenario.py +118 -0
  371. helm/benchmark/scenarios/copyright_scenario.py +35 -1
  372. helm/benchmark/scenarios/covid_dialog_scenario.py +10 -1
  373. helm/benchmark/scenarios/cti_to_mitre_scenario.py +261 -0
  374. helm/benchmark/scenarios/custom_mcqa_scenario.py +1 -1
  375. helm/benchmark/scenarios/czech_bank_qa_scenario.py +148 -0
  376. helm/benchmark/scenarios/decodingtrust_adv_demonstration_scenario.py +190 -0
  377. helm/benchmark/scenarios/decodingtrust_adv_robustness_scenario.py +143 -0
  378. helm/benchmark/scenarios/decodingtrust_fairness_scenario.py +98 -0
  379. helm/benchmark/scenarios/decodingtrust_machine_ethics_scenario.py +344 -0
  380. helm/benchmark/scenarios/decodingtrust_ood_robustness_scenario.py +217 -0
  381. helm/benchmark/scenarios/decodingtrust_privacy_scenario.py +571 -0
  382. helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py +80 -0
  383. helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py +90 -0
  384. helm/benchmark/scenarios/dialogue_scenarios.py +13 -3
  385. helm/benchmark/scenarios/dischargeme_scenario.py +196 -0
  386. helm/benchmark/scenarios/disinformation_scenario.py +32 -1
  387. helm/benchmark/scenarios/dyck_language_scenario.py +25 -1
  388. helm/benchmark/scenarios/echr_judgment_classification_scenario.py +113 -0
  389. helm/benchmark/scenarios/ehr_sql_scenario.py +137 -0
  390. helm/benchmark/scenarios/ehrshot_scenario.py +1541 -0
  391. helm/benchmark/scenarios/enem_challenge_scenario.py +77 -0
  392. helm/benchmark/scenarios/entity_data_imputation_scenario.py +33 -3
  393. helm/benchmark/scenarios/entity_matching_scenario.py +26 -2
  394. helm/benchmark/scenarios/ewok_scenario.py +116 -0
  395. helm/benchmark/scenarios/exams_multilingual_scenario.py +115 -0
  396. helm/benchmark/scenarios/fin_qa_scenario.py +139 -0
  397. helm/benchmark/scenarios/financebench_scenario.py +74 -0
  398. helm/benchmark/scenarios/financial_phrasebank_scenario.py +115 -0
  399. helm/benchmark/scenarios/gold_commodity_news_scenario.py +145 -0
  400. helm/benchmark/scenarios/gpqa_scenario.py +98 -0
  401. helm/benchmark/scenarios/grammar.py +2 -2
  402. helm/benchmark/scenarios/grammar_scenario.py +21 -2
  403. helm/benchmark/scenarios/gsm_scenario.py +31 -1
  404. helm/benchmark/scenarios/harm_bench_gcg_transfer_scenario.py +61 -0
  405. helm/benchmark/scenarios/harm_bench_scenario.py +70 -0
  406. helm/benchmark/scenarios/headqa_scenario.py +158 -0
  407. helm/benchmark/scenarios/healthqa_br_scenario.py +80 -0
  408. helm/benchmark/scenarios/helpdesk_call_summarization_scenario.py +50 -0
  409. helm/benchmark/scenarios/ice_scenario.py +28 -4
  410. helm/benchmark/scenarios/ifeval_scenario.py +71 -0
  411. helm/benchmark/scenarios/image_generation/__init__.py +0 -0
  412. helm/benchmark/scenarios/image_generation/common_syntactic_processes_scenario.py +105 -0
  413. helm/benchmark/scenarios/image_generation/cub200_scenario.py +95 -0
  414. helm/benchmark/scenarios/image_generation/daily_dalle_scenario.py +124 -0
  415. helm/benchmark/scenarios/image_generation/demographic_stereotypes_scenario.py +82 -0
  416. helm/benchmark/scenarios/image_generation/detection_scenario.py +83 -0
  417. helm/benchmark/scenarios/image_generation/draw_bench_scenario.py +74 -0
  418. helm/benchmark/scenarios/image_generation/i2p_scenario.py +57 -0
  419. helm/benchmark/scenarios/image_generation/landing_page_scenario.py +46 -0
  420. helm/benchmark/scenarios/image_generation/logos_scenario.py +223 -0
  421. helm/benchmark/scenarios/image_generation/magazine_cover_scenario.py +91 -0
  422. helm/benchmark/scenarios/image_generation/mental_disorders_scenario.py +46 -0
  423. helm/benchmark/scenarios/image_generation/mscoco_scenario.py +91 -0
  424. helm/benchmark/scenarios/image_generation/paint_skills_scenario.py +72 -0
  425. helm/benchmark/scenarios/image_generation/parti_prompts_scenario.py +94 -0
  426. helm/benchmark/scenarios/image_generation/radiology_scenario.py +42 -0
  427. helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py +52 -0
  428. helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py +124 -0
  429. helm/benchmark/scenarios/image_generation/winoground_scenario.py +62 -0
  430. helm/benchmark/scenarios/imdb_ptbr_scenario.py +60 -0
  431. helm/benchmark/scenarios/imdb_scenario.py +26 -3
  432. helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py +111 -0
  433. helm/benchmark/scenarios/infinite_bench_en_qa_scenario.py +85 -0
  434. helm/benchmark/scenarios/infinite_bench_en_sum_scenario.py +98 -0
  435. helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py +2 -2
  436. helm/benchmark/scenarios/koala_scenario.py +21 -1
  437. helm/benchmark/scenarios/kpi_edgar_scenario.py +172 -0
  438. helm/benchmark/scenarios/legal_contract_summarization_scenario.py +149 -0
  439. helm/benchmark/scenarios/legal_opinion_sentiment_classification_scenario.py +77 -0
  440. helm/benchmark/scenarios/legal_summarization_scenario.py +61 -1
  441. helm/benchmark/scenarios/legal_support_scenario.py +24 -1
  442. helm/benchmark/scenarios/legalbench_scenario.py +45 -3
  443. helm/benchmark/scenarios/lex_glue_scenario.py +23 -2
  444. helm/benchmark/scenarios/lextreme_scenario.py +22 -1
  445. helm/benchmark/scenarios/live_qa_scenario.py +94 -0
  446. helm/benchmark/scenarios/lm_entry_scenario.py +185 -0
  447. helm/benchmark/scenarios/lmkt_scenarios.py +288 -0
  448. helm/benchmark/scenarios/lsat_qa_scenario.py +15 -1
  449. helm/benchmark/scenarios/madinah_qa_scenario.py +73 -0
  450. helm/benchmark/scenarios/math_scenario.py +81 -22
  451. helm/benchmark/scenarios/mbzuai_human_translated_arabic_mmlu.py +68 -0
  452. helm/benchmark/scenarios/me_q_sum_scenario.py +10 -1
  453. helm/benchmark/scenarios/med_dialog_scenario.py +56 -22
  454. helm/benchmark/scenarios/med_mcqa_scenario.py +24 -1
  455. helm/benchmark/scenarios/med_paragraph_simplification_scenario.py +10 -1
  456. helm/benchmark/scenarios/med_qa_scenario.py +30 -1
  457. helm/benchmark/scenarios/medalign_scenario.py +117 -0
  458. helm/benchmark/scenarios/medalign_scenario_helper.py +326 -0
  459. helm/benchmark/scenarios/medbullets_scenario.py +167 -0
  460. helm/benchmark/scenarios/medcalc_bench_scenario.py +149 -0
  461. helm/benchmark/scenarios/medec_scenario.py +148 -0
  462. helm/benchmark/scenarios/medhallu_scenario.py +95 -0
  463. helm/benchmark/scenarios/medhelm/__init__.py +0 -0
  464. helm/benchmark/scenarios/medhelm/judges.yaml +14 -0
  465. helm/benchmark/scenarios/medhelm_configurable_scenario.py +101 -0
  466. helm/benchmark/scenarios/medi_qa_scenario.py +134 -0
  467. helm/benchmark/scenarios/medication_qa_scenario.py +96 -0
  468. helm/benchmark/scenarios/melt_ir_scenario.py +171 -0
  469. helm/benchmark/scenarios/melt_knowledge_scenario.py +246 -0
  470. helm/benchmark/scenarios/melt_lm_scenarios.py +252 -0
  471. helm/benchmark/scenarios/melt_scenarios.py +793 -0
  472. helm/benchmark/scenarios/melt_srn_scenario.py +342 -0
  473. helm/benchmark/scenarios/melt_synthetic_reasoning_scenario.py +222 -0
  474. helm/benchmark/scenarios/melt_translation_scenario.py +152 -0
  475. helm/benchmark/scenarios/mental_health_scenario.py +146 -0
  476. helm/benchmark/scenarios/mimic_bhc_scenario.py +127 -0
  477. helm/benchmark/scenarios/mimic_rrs_scenario.py +121 -0
  478. helm/benchmark/scenarios/mimiciv_billing_code_scenario.py +99 -0
  479. helm/benchmark/scenarios/mmlu_clinical_afr_scenario.py +74 -0
  480. helm/benchmark/scenarios/mmlu_pro_scenario.py +113 -0
  481. helm/benchmark/scenarios/mmlu_scenario.py +32 -1
  482. helm/benchmark/scenarios/mmmlu_scenario.py +85 -0
  483. helm/benchmark/scenarios/msmarco_scenario.py +31 -1
  484. helm/benchmark/scenarios/mtsamples_procedures_scenario.py +166 -0
  485. helm/benchmark/scenarios/mtsamples_replicate_scenario.py +164 -0
  486. helm/benchmark/scenarios/n2c2_ct_matching_scenario.py +297 -0
  487. helm/benchmark/scenarios/narrativeqa_scenario.py +20 -1
  488. helm/benchmark/scenarios/natural_qa_scenario.py +33 -1
  489. helm/benchmark/scenarios/newsqa_scenario.py +1 -1
  490. helm/benchmark/scenarios/oab_exams_scenario.py +57 -0
  491. helm/benchmark/scenarios/omni_math_scenario.py +71 -0
  492. helm/benchmark/scenarios/open_assistant_scenario.py +33 -2
  493. helm/benchmark/scenarios/openai_mrcr_scenario.py +94 -0
  494. helm/benchmark/scenarios/opinions_qa_scenario.py +1 -5
  495. helm/benchmark/scenarios/pubmed_qa_scenario.py +81 -43
  496. helm/benchmark/scenarios/quac_scenario.py +24 -1
  497. helm/benchmark/scenarios/race_based_med_scenario.py +175 -0
  498. helm/benchmark/scenarios/raft_scenario.py +33 -3
  499. helm/benchmark/scenarios/real_toxicity_prompts_scenario.py +14 -1
  500. helm/benchmark/scenarios/ruler_qa_scenario_helper.py +171 -0
  501. helm/benchmark/scenarios/ruler_qa_scenarios.py +128 -0
  502. helm/benchmark/scenarios/scenario.py +44 -1
  503. helm/benchmark/scenarios/seahelm_scenario.py +2295 -0
  504. helm/benchmark/scenarios/self_instruct_scenario.py +29 -1
  505. helm/benchmark/scenarios/shc_bmt_scenario.py +97 -0
  506. helm/benchmark/scenarios/shc_cdi_scenario.py +95 -0
  507. helm/benchmark/scenarios/shc_conf_scenario.py +99 -0
  508. helm/benchmark/scenarios/shc_ent_scenario.py +98 -0
  509. helm/benchmark/scenarios/shc_gip_scenario.py +94 -0
  510. helm/benchmark/scenarios/shc_privacy_scenario.py +100 -0
  511. helm/benchmark/scenarios/shc_proxy_scenario.py +98 -0
  512. helm/benchmark/scenarios/shc_ptbm_scenario.py +104 -0
  513. helm/benchmark/scenarios/shc_sei_scenario.py +94 -0
  514. helm/benchmark/scenarios/shc_sequoia_scenario.py +98 -0
  515. helm/benchmark/scenarios/simple_safety_tests_scenario.py +44 -0
  516. helm/benchmark/scenarios/simple_scenarios.py +122 -1
  517. helm/benchmark/scenarios/situation_prompts.yaml +49 -0
  518. helm/benchmark/scenarios/spider_scenario.py +109 -0
  519. helm/benchmark/scenarios/starr_patient_instructions_scenario.py +119 -0
  520. helm/benchmark/scenarios/summarization_scenario.py +48 -1
  521. helm/benchmark/scenarios/sumosum_scenario.py +157 -0
  522. helm/benchmark/scenarios/synthetic_efficiency_scenario.py +22 -1
  523. helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py +24 -1
  524. helm/benchmark/scenarios/synthetic_reasoning_scenario.py +11 -1
  525. helm/benchmark/scenarios/test_air_bench_scenario.py +27 -0
  526. helm/benchmark/scenarios/test_alghafa_scenario.py +29 -0
  527. helm/benchmark/scenarios/test_alrage_scenario.py +23 -0
  528. helm/benchmark/scenarios/test_arabic_exams_scenario.py +21 -0
  529. helm/benchmark/scenarios/test_aratrust_scenario.py +21 -0
  530. helm/benchmark/scenarios/test_bigcodebench_scenario.py +26 -0
  531. helm/benchmark/scenarios/test_bluex_scenario.py +59 -0
  532. helm/benchmark/scenarios/test_commonsense_scenario.py +21 -0
  533. helm/benchmark/scenarios/test_czech_bank_qa_scenario.py +18 -0
  534. helm/benchmark/scenarios/test_enem_challenge_scenario.py +53 -0
  535. helm/benchmark/scenarios/test_ewok_scenario.py +29 -0
  536. helm/benchmark/scenarios/test_exams_multilingual_scenario.py +29 -0
  537. helm/benchmark/scenarios/test_financebench_scenario.py +26 -0
  538. helm/benchmark/scenarios/test_gold_commodity_news_scenario.py +18 -0
  539. helm/benchmark/scenarios/test_gpqa_scenario.py +44 -0
  540. helm/benchmark/scenarios/test_gsm_scenario.py +31 -0
  541. helm/benchmark/scenarios/test_healtha_br_scenario.py +57 -0
  542. helm/benchmark/scenarios/test_ifeval_scenario.py +36 -0
  543. helm/benchmark/scenarios/test_imdb_ptbr_scenario.py +27 -0
  544. helm/benchmark/scenarios/test_infinite_bench_en_qa_scenario.py +18 -0
  545. helm/benchmark/scenarios/test_infinite_bench_en_sum_scenario.py +31 -0
  546. helm/benchmark/scenarios/test_legalbench_scenario.py +30 -0
  547. helm/benchmark/scenarios/test_math_scenario.py +4 -3
  548. helm/benchmark/scenarios/test_med_qa_scenario.py +30 -0
  549. helm/benchmark/scenarios/test_mmlu_clinical_afr_scenario.py +21 -0
  550. helm/benchmark/scenarios/test_mmlu_pro_scenario.py +53 -0
  551. helm/benchmark/scenarios/test_mmlu_scenario.py +33 -0
  552. helm/benchmark/scenarios/test_narrativeqa_scenario.py +73 -0
  553. helm/benchmark/scenarios/test_oab_exams_scenario.py +51 -0
  554. helm/benchmark/scenarios/test_omni_math_scenario.py +27 -0
  555. helm/benchmark/scenarios/test_scenario.py +6 -3
  556. helm/benchmark/scenarios/test_simple_scenarios.py +50 -0
  557. helm/benchmark/scenarios/test_tweetsentbr_scenario.py +24 -0
  558. helm/benchmark/scenarios/test_wildbench_scenario.py +15 -0
  559. helm/benchmark/scenarios/test_winogrande_afr_scenario.py +19 -0
  560. helm/benchmark/scenarios/thai_exam_scenario.py +239 -0
  561. helm/benchmark/scenarios/the_pile_scenario.py +13 -1
  562. helm/benchmark/scenarios/truthful_qa_scenario.py +26 -2
  563. helm/benchmark/scenarios/tweetsentbr_scenario.py +66 -0
  564. helm/benchmark/scenarios/twitter_aae_scenario.py +20 -1
  565. helm/benchmark/scenarios/unitxt_scenario.py +62 -0
  566. helm/benchmark/scenarios/verifiability_judgment_scenario.py +4 -2
  567. helm/benchmark/scenarios/vicuna_scenario.py +22 -2
  568. helm/benchmark/scenarios/vision_language/a_okvqa_scenario.py +83 -0
  569. helm/benchmark/scenarios/vision_language/bingo_scenario.py +103 -0
  570. helm/benchmark/scenarios/vision_language/blink_scenario.py +140 -0
  571. helm/benchmark/scenarios/vision_language/crossmodal_3600_scenario.py +135 -0
  572. helm/benchmark/scenarios/vision_language/exams_v_scenario.py +104 -0
  573. helm/benchmark/scenarios/vision_language/fair_face_scenario.py +136 -0
  574. helm/benchmark/scenarios/vision_language/flickr30k_scenario.py +74 -0
  575. helm/benchmark/scenarios/vision_language/gqa_scenario.py +91 -0
  576. helm/benchmark/scenarios/vision_language/hateful_memes_scenario.py +94 -0
  577. helm/benchmark/scenarios/vision_language/heim_human_eval_scenario.py +113 -0
  578. helm/benchmark/scenarios/vision_language/image2struct/__init__.py +0 -0
  579. helm/benchmark/scenarios/vision_language/image2struct/chart2csv_scenario.py +55 -0
  580. helm/benchmark/scenarios/vision_language/image2struct/image2struct_scenario.py +225 -0
  581. helm/benchmark/scenarios/vision_language/image2struct/latex_scenario.py +21 -0
  582. helm/benchmark/scenarios/vision_language/image2struct/musicsheet_scenario.py +16 -0
  583. helm/benchmark/scenarios/vision_language/image2struct/utils_latex.py +339 -0
  584. helm/benchmark/scenarios/vision_language/image2struct/webpage/__init__.py +0 -0
  585. helm/benchmark/scenarios/vision_language/image2struct/webpage/driver.py +84 -0
  586. helm/benchmark/scenarios/vision_language/image2struct/webpage/jekyll_server.py +182 -0
  587. helm/benchmark/scenarios/vision_language/image2struct/webpage/utils.py +31 -0
  588. helm/benchmark/scenarios/vision_language/image2struct/webpage_scenario.py +256 -0
  589. helm/benchmark/scenarios/vision_language/math_vista_scenario.py +117 -0
  590. helm/benchmark/scenarios/vision_language/mementos_scenario.py +124 -0
  591. helm/benchmark/scenarios/vision_language/mm_safety_bench_scenario.py +103 -0
  592. helm/benchmark/scenarios/vision_language/mm_star_scenario.py +95 -0
  593. helm/benchmark/scenarios/vision_language/mme_scenario.py +148 -0
  594. helm/benchmark/scenarios/vision_language/mmmu_scenario.py +187 -0
  595. helm/benchmark/scenarios/vision_language/mscoco_captioning_scenario.py +92 -0
  596. helm/benchmark/scenarios/vision_language/mscoco_categorization_scenario.py +117 -0
  597. helm/benchmark/scenarios/vision_language/msr_vtt_scenario.py +75 -0
  598. helm/benchmark/scenarios/vision_language/multipanelvqa_scenario.py +169 -0
  599. helm/benchmark/scenarios/vision_language/originality_scenario.py +35 -0
  600. helm/benchmark/scenarios/vision_language/pairs_scenario.py +247 -0
  601. helm/benchmark/scenarios/vision_language/pope_scenario.py +105 -0
  602. helm/benchmark/scenarios/vision_language/real_world_qa_scenario.py +57 -0
  603. helm/benchmark/scenarios/vision_language/seed_bench_scenario.py +131 -0
  604. helm/benchmark/scenarios/vision_language/unicorn_scenario.py +108 -0
  605. helm/benchmark/scenarios/vision_language/vibe_eval_scenario.py +98 -0
  606. helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py +4 -5
  607. helm/benchmark/scenarios/vision_language/vqa_rad_scenario.py +88 -0
  608. helm/benchmark/scenarios/vision_language/vqa_scenario.py +8 -4
  609. helm/benchmark/scenarios/wikifact_scenario.py +31 -1
  610. helm/benchmark/scenarios/wikitext_103_scenario.py +1 -1
  611. helm/benchmark/scenarios/wildbench_scenario.py +101 -0
  612. helm/benchmark/scenarios/winogrande_afr_scenario.py +78 -0
  613. helm/benchmark/scenarios/wmt_14_scenario.py +33 -2
  614. helm/benchmark/scenarios/xstest_scenario.py +35 -0
  615. helm/benchmark/server.py +32 -2
  616. helm/benchmark/slurm_jobs.py +1 -2
  617. helm/benchmark/slurm_runner.py +78 -50
  618. helm/benchmark/static/schema_air_bench.yaml +3149 -0
  619. helm/benchmark/static/schema_arabic.yaml +271 -0
  620. helm/benchmark/static/schema_audio.yaml +763 -0
  621. helm/benchmark/static/schema_autobencher.yaml +150 -0
  622. helm/benchmark/static/schema_call_center.yaml +269 -0
  623. helm/benchmark/static/schema_capabilities.yaml +254 -0
  624. helm/benchmark/static/schema_classic.yaml +259 -1140
  625. helm/benchmark/static/schema_cleva.yaml +768 -0
  626. helm/benchmark/static/schema_czech_bank.yaml +148 -0
  627. helm/benchmark/static/schema_decodingtrust.yaml +444 -0
  628. helm/benchmark/static/schema_enem_challenge.yaml +146 -0
  629. helm/benchmark/static/schema_enterprise.yaml +319 -0
  630. helm/benchmark/static/schema_ewok.yaml +367 -0
  631. helm/benchmark/static/schema_finance.yaml +191 -0
  632. helm/benchmark/static/schema_heim.yaml +1389 -0
  633. helm/benchmark/static/schema_image2struct.yaml +588 -0
  634. helm/benchmark/static/schema_instruction_following.yaml +161 -0
  635. helm/benchmark/static/schema_legal.yaml +566 -0
  636. helm/benchmark/static/schema_lite.yaml +3 -286
  637. helm/benchmark/static/schema_long_context.yaml +282 -0
  638. helm/benchmark/static/schema_medhelm.yaml +1176 -0
  639. helm/benchmark/static/schema_melt.yaml +1257 -0
  640. helm/benchmark/static/schema_mmlu.yaml +1449 -0
  641. helm/benchmark/static/schema_mmlu_winogrande_afr.yaml +1045 -0
  642. helm/benchmark/static/schema_safety.yaml +283 -0
  643. helm/benchmark/static/schema_seahelm.yaml +723 -0
  644. helm/benchmark/static/schema_slp.yaml +219 -0
  645. helm/benchmark/static/schema_slphelm.yaml +162 -0
  646. helm/benchmark/static/schema_social_audio.yaml +224 -0
  647. helm/benchmark/static/schema_sql.yaml +171 -0
  648. helm/benchmark/static/schema_thai.yaml +244 -0
  649. helm/benchmark/static/schema_torr.yaml +474 -0
  650. helm/benchmark/static/schema_tweetsentbr.yaml +146 -0
  651. helm/benchmark/static/schema_unitxt.yaml +370 -0
  652. helm/benchmark/static/schema_vhelm.yaml +933 -0
  653. helm/benchmark/static/schema_vhelm_lite.yaml +109 -0
  654. helm/benchmark/static/schema_video.yaml +219 -0
  655. helm/benchmark/static_build/assets/air-overview-DpBbyagA.png +0 -0
  656. helm/benchmark/static_build/assets/audio-table-Dn5NMMeJ.png +0 -0
  657. helm/benchmark/static_build/assets/heim-logo-BJtQlEbV.png +0 -0
  658. helm/benchmark/static_build/assets/helm-safety-COfndXuS.png +0 -0
  659. helm/benchmark/static_build/assets/helmhero-D9TvmJsp.png +0 -0
  660. helm/benchmark/static_build/assets/index-oIeiQW2g.css +1 -0
  661. helm/benchmark/static_build/assets/index-qOFpOyHb.js +10 -0
  662. helm/benchmark/static_build/assets/medhelm-overview-CND0EIsy.png +0 -0
  663. helm/benchmark/static_build/assets/medhelm-v1-overview-Cu2tphBB.png +0 -0
  664. helm/benchmark/static_build/assets/overview-BwypNWnk.png +0 -0
  665. helm/benchmark/static_build/assets/process-flow-DWDJC733.png +0 -0
  666. helm/benchmark/static_build/assets/react-BteFIppM.js +85 -0
  667. helm/benchmark/static_build/assets/recharts-DxuQtTOs.js +97 -0
  668. helm/benchmark/static_build/assets/tremor-DR4fE7ko.js +10 -0
  669. helm/benchmark/static_build/assets/vhelm-aspects-NiDQofvP.png +0 -0
  670. helm/benchmark/static_build/assets/vhelm-framework-NxJE4fdA.png +0 -0
  671. helm/benchmark/static_build/assets/vhelm-model-ypCL5Yvq.png +0 -0
  672. helm/benchmark/static_build/config.js +4 -0
  673. helm/benchmark/static_build/index.html +19 -0
  674. helm/benchmark/test_data_preprocessor.py +3 -3
  675. helm/benchmark/test_run_expander.py +1 -1
  676. helm/benchmark/window_services/default_window_service.py +3 -45
  677. helm/benchmark/window_services/encoder_decoder_window_service.py +4 -15
  678. helm/benchmark/window_services/ice_window_service.py +1 -35
  679. helm/benchmark/window_services/image_generation/__init__.py +0 -0
  680. helm/benchmark/window_services/image_generation/clip_window_service.py +13 -0
  681. helm/benchmark/window_services/image_generation/lexica_search_window_service.py +9 -0
  682. helm/benchmark/window_services/image_generation/openai_dalle_window_service.py +9 -0
  683. helm/benchmark/window_services/image_generation/test_clip_window_service.py +29 -0
  684. helm/benchmark/window_services/image_generation/test_openai_dalle_window_service.py +30 -0
  685. helm/benchmark/window_services/local_window_service.py +22 -5
  686. helm/benchmark/window_services/test_anthropic_window_service.py +5 -4
  687. helm/benchmark/window_services/test_bloom_window_service.py +5 -4
  688. helm/benchmark/window_services/test_flan_t5_window_service.py +2 -1
  689. helm/benchmark/window_services/test_gpt2_window_service.py +9 -4
  690. helm/benchmark/window_services/test_gpt4_window_service.py +10 -4
  691. helm/benchmark/window_services/test_gptj_window_service.py +11 -5
  692. helm/benchmark/window_services/test_gptneox_window_service.py +6 -5
  693. helm/benchmark/window_services/test_openai_window_service.py +18 -12
  694. helm/benchmark/window_services/test_opt_window_service.py +6 -5
  695. helm/benchmark/window_services/test_palmyra_window_service.py +5 -4
  696. helm/benchmark/window_services/test_t0pp_window_service.py +5 -4
  697. helm/benchmark/window_services/test_t511b_window_service.py +5 -4
  698. helm/benchmark/window_services/test_ul2_window_service.py +5 -4
  699. helm/benchmark/window_services/test_utils.py +6 -6
  700. helm/benchmark/window_services/test_yalm_window_service.py +5 -4
  701. helm/benchmark/window_services/tokenizer_service.py +7 -13
  702. helm/benchmark/window_services/window_service.py +42 -0
  703. helm/benchmark/window_services/window_service_factory.py +4 -1
  704. helm/benchmark/window_services/yalm_window_service.py +1 -28
  705. helm/clients/__init__.py +0 -0
  706. helm/{proxy/clients → clients}/ai21_client.py +78 -12
  707. helm/clients/aleph_alpha_client.py +114 -0
  708. helm/{proxy/clients → clients}/anthropic_client.py +304 -21
  709. helm/clients/audio_language/__init__.py +0 -0
  710. helm/clients/audio_language/diva_llama_client.py +122 -0
  711. helm/clients/audio_language/llama_omni/arguments.py +61 -0
  712. helm/clients/audio_language/llama_omni/constants.py +9 -0
  713. helm/clients/audio_language/llama_omni/conversation.py +213 -0
  714. helm/clients/audio_language/llama_omni/model/__init__.py +0 -0
  715. helm/clients/audio_language/llama_omni/model/builder.py +88 -0
  716. helm/clients/audio_language/llama_omni/model/language_model/omni_speech2s_llama.py +190 -0
  717. helm/clients/audio_language/llama_omni/model/language_model/omni_speech_llama.py +118 -0
  718. helm/clients/audio_language/llama_omni/model/omni_speech_arch.py +249 -0
  719. helm/clients/audio_language/llama_omni/model/speech_encoder/builder.py +9 -0
  720. helm/clients/audio_language/llama_omni/model/speech_encoder/speech_encoder.py +27 -0
  721. helm/clients/audio_language/llama_omni/model/speech_generator/builder.py +9 -0
  722. helm/clients/audio_language/llama_omni/model/speech_generator/generation.py +622 -0
  723. helm/clients/audio_language/llama_omni/model/speech_generator/speech_generator.py +104 -0
  724. helm/clients/audio_language/llama_omni/model/speech_projector/builder.py +9 -0
  725. helm/clients/audio_language/llama_omni/model/speech_projector/speech_projector.py +27 -0
  726. helm/clients/audio_language/llama_omni/preprocess.py +295 -0
  727. helm/clients/audio_language/llama_omni/utils.py +202 -0
  728. helm/clients/audio_language/llama_omni_client.py +199 -0
  729. helm/clients/audio_language/qwen2_5_omni_client.py +210 -0
  730. helm/clients/audio_language/qwen2_audiolm_client.py +191 -0
  731. helm/clients/audio_language/qwen_audiolm_client.py +153 -0
  732. helm/clients/audio_language/qwen_omni/configuration_qwen2_5_omni.py +519 -0
  733. helm/clients/audio_language/qwen_omni/modeling_qwen2_5_omni.py +4308 -0
  734. helm/clients/audio_language/qwen_omni/processing_qwen2_5_omni.py +270 -0
  735. helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/__init__.py +0 -0
  736. helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/__init__.py +8 -0
  737. helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/audio_process.py +56 -0
  738. helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/vision_process.py +380 -0
  739. helm/clients/audio_language/test.py +62 -0
  740. helm/{proxy/clients → clients}/auto_client.py +72 -31
  741. helm/clients/azure_openai_client.py +55 -0
  742. helm/clients/bedrock_client.py +381 -0
  743. helm/clients/bedrock_utils.py +105 -0
  744. helm/{proxy/clients → clients}/client.py +92 -17
  745. helm/clients/clip_score_client.py +49 -0
  746. helm/clients/clip_scorers/__init__.py +0 -0
  747. helm/clients/clip_scorers/base_clip_scorer.py +18 -0
  748. helm/clients/clip_scorers/clip_scorer.py +50 -0
  749. helm/clients/clip_scorers/multilingual_clip_scorer.py +50 -0
  750. helm/{proxy/clients → clients}/cohere_client.py +105 -14
  751. helm/clients/dspy_client.py +135 -0
  752. helm/clients/gcs_client.py +82 -0
  753. helm/{proxy/clients → clients}/google_client.py +8 -6
  754. helm/clients/google_translate_client.py +35 -0
  755. helm/clients/grok_client.py +36 -0
  756. helm/{proxy/clients → clients}/http_model_client.py +8 -8
  757. helm/{proxy/clients → clients}/huggingface_client.py +157 -86
  758. helm/clients/huggingface_pipeline_client.py +138 -0
  759. helm/clients/ibm_client.py +269 -0
  760. helm/clients/image_generation/__init__.py +0 -0
  761. helm/clients/image_generation/adobe_vision_client.py +80 -0
  762. helm/clients/image_generation/aleph_alpha_image_generation_client.py +100 -0
  763. helm/clients/image_generation/cogview2/__init__.py +0 -0
  764. helm/clients/image_generation/cogview2/coglm_strategy.py +96 -0
  765. helm/clients/image_generation/cogview2/coglm_utils.py +82 -0
  766. helm/clients/image_generation/cogview2/sr_pipeline/__init__.py +15 -0
  767. helm/clients/image_generation/cogview2/sr_pipeline/direct_sr.py +99 -0
  768. helm/clients/image_generation/cogview2/sr_pipeline/dsr_model.py +254 -0
  769. helm/clients/image_generation/cogview2/sr_pipeline/dsr_sampling.py +190 -0
  770. helm/clients/image_generation/cogview2/sr_pipeline/iterative_sr.py +144 -0
  771. helm/clients/image_generation/cogview2/sr_pipeline/itersr_model.py +269 -0
  772. helm/clients/image_generation/cogview2/sr_pipeline/itersr_sampling.py +120 -0
  773. helm/clients/image_generation/cogview2/sr_pipeline/sr_group.py +42 -0
  774. helm/clients/image_generation/cogview2_client.py +192 -0
  775. helm/clients/image_generation/dalle2_client.py +194 -0
  776. helm/clients/image_generation/dalle3_client.py +108 -0
  777. helm/clients/image_generation/dalle_mini/__init__.py +3 -0
  778. helm/clients/image_generation/dalle_mini/data.py +442 -0
  779. helm/clients/image_generation/dalle_mini/model/__init__.py +5 -0
  780. helm/clients/image_generation/dalle_mini/model/configuration.py +175 -0
  781. helm/clients/image_generation/dalle_mini/model/modeling.py +1834 -0
  782. helm/clients/image_generation/dalle_mini/model/partitions.py +84 -0
  783. helm/clients/image_generation/dalle_mini/model/processor.py +63 -0
  784. helm/clients/image_generation/dalle_mini/model/text.py +251 -0
  785. helm/clients/image_generation/dalle_mini/model/tokenizer.py +9 -0
  786. helm/clients/image_generation/dalle_mini/model/utils.py +29 -0
  787. helm/clients/image_generation/dalle_mini/vqgan_jax/__init__.py +1 -0
  788. helm/clients/image_generation/dalle_mini/vqgan_jax/configuration_vqgan.py +40 -0
  789. helm/clients/image_generation/dalle_mini/vqgan_jax/convert_pt_model_to_jax.py +107 -0
  790. helm/clients/image_generation/dalle_mini/vqgan_jax/modeling_flax_vqgan.py +610 -0
  791. helm/clients/image_generation/dalle_mini_client.py +191 -0
  792. helm/clients/image_generation/deep_floyd_client.py +80 -0
  793. helm/clients/image_generation/huggingface_diffusers_client.py +250 -0
  794. helm/clients/image_generation/image_generation_client_utils.py +9 -0
  795. helm/clients/image_generation/lexica_client.py +88 -0
  796. helm/clients/image_generation/mindalle/__init__.py +0 -0
  797. helm/clients/image_generation/mindalle/models/__init__.py +216 -0
  798. helm/clients/image_generation/mindalle/models/stage1/__init__.py +0 -0
  799. helm/clients/image_generation/mindalle/models/stage1/layers.py +312 -0
  800. helm/clients/image_generation/mindalle/models/stage1/vqgan.py +103 -0
  801. helm/clients/image_generation/mindalle/models/stage2/__init__.py +0 -0
  802. helm/clients/image_generation/mindalle/models/stage2/layers.py +144 -0
  803. helm/clients/image_generation/mindalle/models/stage2/transformer.py +268 -0
  804. helm/clients/image_generation/mindalle/models/tokenizer.py +30 -0
  805. helm/clients/image_generation/mindalle/utils/__init__.py +3 -0
  806. helm/clients/image_generation/mindalle/utils/config.py +129 -0
  807. helm/clients/image_generation/mindalle/utils/sampling.py +149 -0
  808. helm/clients/image_generation/mindalle/utils/utils.py +89 -0
  809. helm/clients/image_generation/mindalle_client.py +116 -0
  810. helm/clients/image_generation/nudity_check_client.py +64 -0
  811. helm/clients/image_generation/together_image_generation_client.py +113 -0
  812. helm/{proxy/clients → clients}/lit_gpt_client.py +6 -6
  813. helm/{proxy/clients → clients}/megatron_client.py +7 -5
  814. helm/clients/mistral_client.py +180 -0
  815. helm/clients/moderation_api_client.py +111 -0
  816. helm/clients/nvidia_nim_client.py +32 -0
  817. helm/clients/open_lm_client.py +43 -0
  818. helm/clients/openai_client.py +604 -0
  819. helm/clients/openai_responses_client.py +200 -0
  820. helm/clients/openrouter_client.py +31 -0
  821. helm/{proxy/clients → clients}/palmyra_client.py +31 -14
  822. helm/{proxy/clients → clients}/perspective_api_client.py +18 -14
  823. helm/clients/reka_client.py +190 -0
  824. helm/clients/simple_client.py +64 -0
  825. helm/clients/stanfordhealthcare_azure_openai_client.py +58 -0
  826. helm/clients/stanfordhealthcare_claude_client.py +31 -0
  827. helm/clients/stanfordhealthcare_google_client.py +43 -0
  828. helm/clients/stanfordhealthcare_http_model_client.py +95 -0
  829. helm/clients/stanfordhealthcare_openai_client.py +62 -0
  830. helm/clients/stanfordhealthcare_shc_openai_client.py +42 -0
  831. helm/{proxy/clients → clients}/test_auto_client.py +13 -15
  832. helm/clients/test_client.py +98 -0
  833. helm/{proxy/clients → clients}/test_huggingface_client.py +31 -16
  834. helm/clients/test_openrouter_client.py +69 -0
  835. helm/clients/test_simple_client.py +19 -0
  836. helm/clients/test_together_client.py +184 -0
  837. helm/clients/together_client.py +599 -0
  838. helm/clients/upstage_client.py +23 -0
  839. helm/clients/vertexai_client.py +488 -0
  840. helm/clients/vision_language/__init__.py +0 -0
  841. helm/clients/vision_language/huggingface_vision2seq_client.py +148 -0
  842. helm/clients/vision_language/huggingface_vlm_client.py +114 -0
  843. helm/{proxy/clients → clients}/vision_language/idefics_client.py +61 -51
  844. helm/clients/vision_language/open_flamingo/__init__.py +2 -0
  845. helm/clients/vision_language/open_flamingo/src/__init__.py +0 -0
  846. helm/clients/vision_language/open_flamingo/src/factory.py +147 -0
  847. helm/clients/vision_language/open_flamingo/src/flamingo.py +337 -0
  848. helm/clients/vision_language/open_flamingo/src/flamingo_lm.py +155 -0
  849. helm/clients/vision_language/open_flamingo/src/helpers.py +267 -0
  850. helm/clients/vision_language/open_flamingo/src/utils.py +47 -0
  851. helm/clients/vision_language/open_flamingo_client.py +155 -0
  852. helm/clients/vision_language/paligemma_client.py +147 -0
  853. helm/clients/vision_language/palmyra_vision_client.py +101 -0
  854. helm/clients/vision_language/qwen2_vlm_client.py +189 -0
  855. helm/clients/vision_language/qwen_vlm_client.py +174 -0
  856. helm/clients/vllm_client.py +80 -0
  857. helm/clients/vllm_granite_thinking_client.py +56 -0
  858. helm/clients/writer_client.py +105 -0
  859. helm/clients/yi_client.py +28 -0
  860. helm/common/audio_utils.py +111 -0
  861. helm/common/cache.py +23 -33
  862. helm/common/cache_backend_config.py +47 -0
  863. helm/common/clip_score_request.py +41 -0
  864. helm/common/context.py +80 -0
  865. helm/common/credentials_utils.py +5 -5
  866. helm/common/critique_request.py +10 -2
  867. helm/common/file_caches/__init__.py +0 -0
  868. helm/common/file_caches/file_cache.py +16 -0
  869. helm/common/file_caches/local_file_cache.py +61 -0
  870. helm/common/file_caches/test_local_file_cache.py +25 -0
  871. helm/common/file_upload_request.py +27 -0
  872. helm/common/general.py +10 -3
  873. helm/common/hierarchical_logger.py +124 -12
  874. helm/common/image_generation_parameters.py +25 -0
  875. helm/common/images_utils.py +60 -5
  876. helm/common/key_value_store.py +41 -10
  877. helm/common/local_context.py +140 -0
  878. helm/common/media_object.py +14 -1
  879. helm/common/moderations_api_request.py +71 -0
  880. helm/common/mongo_key_value_store.py +8 -7
  881. helm/common/multimodal_request_utils.py +57 -0
  882. helm/common/nudity_check_request.py +29 -0
  883. helm/common/object_spec.py +23 -8
  884. helm/common/optional_dependencies.py +1 -1
  885. helm/common/reeval_parameters.py +12 -0
  886. helm/common/remote_context.py +61 -0
  887. helm/common/request.py +45 -19
  888. helm/common/response_format.py +18 -0
  889. helm/common/test_cache.py +1 -48
  890. helm/common/test_general.py +10 -0
  891. helm/common/test_logging.py +94 -0
  892. helm/common/test_media_object.py +1 -1
  893. helm/common/tokenization_request.py +1 -10
  894. helm/config/model_deployments.yaml +4713 -1005
  895. helm/config/model_metadata.yaml +4045 -255
  896. helm/config/tokenizer_configs.yaml +1091 -50
  897. helm/proxy/accounts.py +31 -4
  898. helm/proxy/cli.py +6 -4
  899. helm/proxy/critique/mechanical_turk_critique_importer.py +3 -0
  900. helm/proxy/critique/mechanical_turk_utils.py +1 -1
  901. helm/proxy/critique/model_critique_client.py +40 -10
  902. helm/proxy/example_queries.py +33 -28
  903. helm/proxy/retry.py +5 -0
  904. helm/proxy/server.py +82 -18
  905. helm/proxy/services/remote_service.py +32 -7
  906. helm/proxy/services/server_service.py +71 -69
  907. helm/proxy/services/service.py +30 -6
  908. helm/proxy/services/test_remote_service.py +6 -5
  909. helm/proxy/services/test_service.py +1 -13
  910. helm/proxy/static/help.html +99 -0
  911. helm/proxy/static/index.css +61 -0
  912. helm/proxy/static/index.html +40 -0
  913. helm/proxy/static/index.js +462 -0
  914. helm/proxy/test_accounts.py +32 -0
  915. helm/proxy/test_retry.py +1 -1
  916. helm/proxy/token_counters/auto_token_counter.py +37 -37
  917. helm/proxy/token_counters/test_auto_token_counter.py +164 -0
  918. helm/proxy/token_counters/token_counter.py +3 -5
  919. helm/tokenizers/__init__.py +0 -0
  920. helm/tokenizers/ai21_tokenizer.py +52 -0
  921. helm/{proxy/tokenizers → tokenizers}/aleph_alpha_tokenizer.py +1 -1
  922. helm/{proxy/tokenizers → tokenizers}/auto_tokenizer.py +9 -12
  923. helm/{proxy/tokenizers → tokenizers}/caching_tokenizer.py +2 -30
  924. helm/tokenizers/cohere_tokenizer.py +50 -0
  925. helm/tokenizers/grok_tokenizer.py +55 -0
  926. helm/{proxy/tokenizers → tokenizers}/http_model_tokenizer.py +4 -4
  927. helm/{proxy/tokenizers → tokenizers}/huggingface_tokenizer.py +44 -41
  928. helm/{proxy/tokenizers → tokenizers}/lit_gpt_tokenizer.py +1 -1
  929. helm/tokenizers/simple_tokenizer.py +33 -0
  930. helm/tokenizers/test_ai21_tokenizer.py +48 -0
  931. helm/{proxy/tokenizers → tokenizers}/test_anthropic_tokenizer.py +6 -2
  932. helm/tokenizers/test_cohere_tokenizer.py +39 -0
  933. helm/tokenizers/test_grok_tokenizer.py +33 -0
  934. helm/{proxy/tokenizers → tokenizers}/test_huggingface_tokenizer.py +9 -2
  935. helm/tokenizers/test_simple_tokenizer.py +33 -0
  936. helm/{proxy/tokenizers → tokenizers}/test_yalm_tokenizer.py +1 -1
  937. helm/{proxy/tokenizers → tokenizers}/tiktoken_tokenizer.py +1 -1
  938. helm/{proxy/tokenizers → tokenizers}/tokenizer.py +3 -1
  939. helm/{proxy/tokenizers → tokenizers}/vertexai_tokenizer.py +1 -1
  940. helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer.py +8 -6
  941. helm/tokenizers/yalm_tokenizer_data/__init__.py +0 -0
  942. helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/test_yalm_tokenizer.py +1 -1
  943. helm/tokenizers/yalm_tokenizer_data/voc_100b.sp +0 -0
  944. helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/yalm_tokenizer.py +1 -1
  945. crfm_helm-0.4.0.dist-info/METADATA +0 -264
  946. crfm_helm-0.4.0.dist-info/RECORD +0 -397
  947. helm/benchmark/data_overlap/data_overlap_spec.py +0 -86
  948. helm/benchmark/data_overlap/export_scenario_text.py +0 -119
  949. helm/benchmark/data_overlap/light_scenario.py +0 -60
  950. helm/benchmark/metrics/numeracy_metrics.py +0 -72
  951. helm/benchmark/metrics/test_numeracy_metrics.py +0 -95
  952. helm/benchmark/run_specs.py +0 -2762
  953. helm/benchmark/scenarios/numeracy_scenario.py +0 -784
  954. helm/benchmark/static/benchmarking.css +0 -156
  955. helm/benchmark/static/benchmarking.js +0 -1705
  956. helm/benchmark/static/config.js +0 -3
  957. helm/benchmark/static/images/helm-logo.png +0 -0
  958. helm/benchmark/static/images/language-model-helm.png +0 -0
  959. helm/benchmark/static/images/organizations/ai21.png +0 -0
  960. helm/benchmark/static/images/organizations/anthropic.png +0 -0
  961. helm/benchmark/static/images/organizations/bigscience.png +0 -0
  962. helm/benchmark/static/images/organizations/cohere.png +0 -0
  963. helm/benchmark/static/images/organizations/eleutherai.png +0 -0
  964. helm/benchmark/static/images/organizations/google.png +0 -0
  965. helm/benchmark/static/images/organizations/meta.png +0 -0
  966. helm/benchmark/static/images/organizations/microsoft.png +0 -0
  967. helm/benchmark/static/images/organizations/nvidia.png +0 -0
  968. helm/benchmark/static/images/organizations/openai.png +0 -0
  969. helm/benchmark/static/images/organizations/together.png +0 -0
  970. helm/benchmark/static/images/organizations/tsinghua-keg.png +0 -0
  971. helm/benchmark/static/images/organizations/yandex.png +0 -0
  972. helm/benchmark/static/images/scenarios-by-metrics.png +0 -0
  973. helm/benchmark/static/images/taxonomy-scenarios.png +0 -0
  974. helm/benchmark/static/index.html +0 -68
  975. helm/benchmark/static/json-urls.js +0 -69
  976. helm/benchmark/static/plot-captions.js +0 -27
  977. helm/benchmark/static/utils.js +0 -285
  978. helm/benchmark/test_model_deployment_definition.py +0 -92
  979. helm/benchmark/test_model_properties.py +0 -1570
  980. helm/benchmark/vlm_run_specs.py +0 -97
  981. helm/benchmark/window_services/ai21_window_service.py +0 -258
  982. helm/benchmark/window_services/cohere_window_service.py +0 -163
  983. helm/benchmark/window_services/flan_t5_window_service.py +0 -29
  984. helm/benchmark/window_services/gpt2_window_service.py +0 -32
  985. helm/benchmark/window_services/huggingface_window_service.py +0 -60
  986. helm/benchmark/window_services/t0pp_window_service.py +0 -35
  987. helm/benchmark/window_services/t511b_window_service.py +0 -30
  988. helm/benchmark/window_services/test_ai21_window_service.py +0 -163
  989. helm/benchmark/window_services/test_cohere_window_service.py +0 -74
  990. helm/benchmark/window_services/test_cohere_window_service_utils.py +0 -8328
  991. helm/benchmark/window_services/test_ice_window_service.py +0 -326
  992. helm/benchmark/window_services/test_mt_nlg_window_service.py +0 -48
  993. helm/benchmark/window_services/ul2_window_service.py +0 -30
  994. helm/benchmark/window_services/wider_ai21_window_service.py +0 -24
  995. helm/common/cache_utils.py +0 -14
  996. helm/proxy/clients/aleph_alpha_client.py +0 -95
  997. helm/proxy/clients/goose_ai_client.py +0 -99
  998. helm/proxy/clients/microsoft_client.py +0 -180
  999. helm/proxy/clients/openai_client.py +0 -206
  1000. helm/proxy/clients/simple_client.py +0 -60
  1001. helm/proxy/clients/test_client.py +0 -49
  1002. helm/proxy/clients/test_together_client.py +0 -97
  1003. helm/proxy/clients/together_client.py +0 -334
  1004. helm/proxy/clients/vertexai_client.py +0 -115
  1005. helm/proxy/token_counters/ai21_token_counter.py +0 -20
  1006. helm/proxy/token_counters/cohere_token_counter.py +0 -13
  1007. helm/proxy/token_counters/free_token_counter.py +0 -12
  1008. helm/proxy/token_counters/gooseai_token_counter.py +0 -24
  1009. helm/proxy/token_counters/openai_token_counter.py +0 -22
  1010. helm/proxy/token_counters/test_ai21_token_counter.py +0 -88
  1011. helm/proxy/token_counters/test_openai_token_counter.py +0 -81
  1012. helm/proxy/tokenizers/ai21_tokenizer.py +0 -60
  1013. helm/proxy/tokenizers/anthropic_tokenizer.py +0 -52
  1014. helm/proxy/tokenizers/cohere_tokenizer.py +0 -83
  1015. helm/proxy/tokenizers/ice_tokenizer.py +0 -30
  1016. helm/proxy/tokenizers/simple_tokenizer.py +0 -32
  1017. helm/proxy/tokenizers/test_ice_tokenizer.py +0 -57
  1018. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.10.dist-info}/entry_points.txt +0 -0
  1019. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.10.dist-info/licenses}/LICENSE +0 -0
  1020. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.10.dist-info}/top_level.txt +0 -0
  1021. /helm/benchmark/{data_overlap → annotation}/__init__.py +0 -0
  1022. /helm/{proxy/clients → benchmark/annotation/image2struct}/__init__.py +0 -0
  1023. /helm/{proxy/clients/vision_language → benchmark/metrics/ifeval}/__init__.py +0 -0
  1024. /helm/{proxy/tokenizers → benchmark/metrics/image_generation}/__init__.py +0 -0
  1025. /helm/{proxy/tokenizers/yalm_tokenizer_data → benchmark/metrics/image_generation/detectors}/__init__.py +0 -0
  1026. /helm/benchmark/{static/images/crfm-logo.png → static_build/assets/crfm-logo-Du4T1uWZ.png} +0 -0
  1027. /helm/benchmark/{static/images/helm-logo-simple.png → static_build/assets/helm-logo-simple-DzOhNN41.png} +0 -0
  1028. /helm/{proxy/clients → clients}/ai21_utils.py +0 -0
  1029. /helm/{proxy/clients → clients}/cohere_utils.py +0 -0
  1030. /helm/{proxy/clients → clients}/lit_gpt_generate.py +0 -0
  1031. /helm/{proxy/clients → clients}/toxicity_classifier_client.py +0 -0
  1032. /helm/{benchmark → proxy}/static/general.js +0 -0
  1033. /helm/{benchmark → proxy}/static/info-icon.png +0 -0
helm/benchmark/test_model_properties.py DELETED
@@ -1,1570 +0,0 @@
1
- """Temporary test for preserving invariants during the model / tokenizer / window service refactor.
2
-
3
- Delete this after the refactor is done."""
4
-
5
- import pytest
6
- from tempfile import TemporaryDirectory
7
- from typing import Any
8
- from helm.benchmark.config_registry import register_builtin_configs_from_helm_package
9
- from helm.benchmark.model_deployment_registry import (
10
- ClientSpec,
11
- ModelDeployment,
12
- WindowServiceSpec,
13
- get_model_deployment,
14
- )
15
- from helm.benchmark.model_metadata_registry import get_model_metadata, ModelMetadata
16
- from helm.benchmark.tokenizer_config_registry import TokenizerConfig, TokenizerSpec
17
- from helm.benchmark.window_services.test_utils import get_tokenizer_service
18
-
19
- from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
20
- from helm.proxy.clients.auto_client import AutoClient
21
- from helm.proxy.tokenizers.auto_tokenizer import AutoTokenizer
22
- from collections import defaultdict
23
-
24
-
25
- _BUILT_IN_TOKENIZER_CONFIGS = [
26
- TokenizerConfig(
27
- name="simple/model1",
28
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.simple_tokenizer.SimpleTokenizer"),
29
- end_of_text_token="</s>",
30
- prefix_token="<s>",
31
- ),
32
- TokenizerConfig(
33
- name="neurips/local",
34
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.http_model_tokenizer.HTTPModelTokenizer"),
35
- end_of_text_token="<|endoftext|>",
36
- prefix_token="<|endoftext|>",
37
- ),
38
- TokenizerConfig(
39
- name="ai21/j1",
40
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.ai21_tokenizer.AI21Tokenizer"),
41
- end_of_text_token=" ",
42
- prefix_token="",
43
- ),
44
- TokenizerConfig(
45
- name="AlephAlpha/luminous-base",
46
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"),
47
- end_of_text_token="",
48
- prefix_token="",
49
- ),
50
- TokenizerConfig(
51
- name="AlephAlpha/luminous-extended",
52
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"),
53
- end_of_text_token="",
54
- prefix_token="",
55
- ),
56
- TokenizerConfig(
57
- name="AlephAlpha/luminous-supreme",
58
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"),
59
- end_of_text_token="",
60
- prefix_token="",
61
- ),
62
- TokenizerConfig(
63
- name="huggingface/gpt2",
64
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
65
- end_of_text_token="<|endoftext|>",
66
- prefix_token="<|endoftext|>",
67
- ),
68
- TokenizerConfig(
69
- name="microsoft/gpt2",
70
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
71
- end_of_text_token="<|endoftext|>",
72
- prefix_token="<<",
73
- ),
74
- TokenizerConfig(
75
- name="writer/gpt2",
76
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
77
- end_of_text_token="",
78
- prefix_token="",
79
- ),
80
- TokenizerConfig(
81
- name="anthropic/claude",
82
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.anthropic_tokenizer.AnthropicTokenizer"),
83
- end_of_text_token="<|endoftext|>",
84
- prefix_token="<|endoftext|>",
85
- ),
86
- TokenizerConfig(
87
- name="bigscience/bloom",
88
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
89
- end_of_text_token="</s>",
90
- prefix_token="</s>",
91
- ),
92
- TokenizerConfig(
93
- name="bigscience/T0pp",
94
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
95
- end_of_text_token="</s>",
96
- prefix_token="",
97
- ),
98
- TokenizerConfig(
99
- name="cohere/cohere",
100
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.cohere_tokenizer.CohereTokenizer"),
101
- end_of_text_token="",
102
- prefix_token=":",
103
- ),
104
- TokenizerConfig(
105
- name="EleutherAI/gpt-j-6B",
106
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
107
- end_of_text_token="<|endoftext|>",
108
- prefix_token="<|endoftext|>",
109
- ),
110
- TokenizerConfig(
111
- name="EleutherAI/gpt-neox-20b",
112
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
113
- end_of_text_token="<|endoftext|>",
114
- prefix_token="<|endoftext|>",
115
- ),
116
- TokenizerConfig(
117
- name="hf-internal-testing/llama-tokenizer",
118
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
119
- end_of_text_token="</s>",
120
- prefix_token="<s>",
121
- ),
122
- TokenizerConfig(
123
- name="meta-llama/Llama-2-7b-hf",
124
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
125
- end_of_text_token="</s>",
126
- prefix_token="<s>",
127
- ),
128
- TokenizerConfig(
129
- name="mistralai/Mistral-7B-v0.1",
130
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
131
- end_of_text_token="</s>",
132
- prefix_token="<s>",
133
- ),
134
- TokenizerConfig(
135
- name="tiiuae/falcon-7b",
136
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
137
- end_of_text_token="<|endoftext|>",
138
- prefix_token="",
139
- ),
140
- TokenizerConfig(
141
- name="bigcode/santacoder",
142
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
143
- end_of_text_token="<|endoftext|>",
144
- prefix_token="<|endoftext|>",
145
- ),
146
- TokenizerConfig(
147
- name="bigcode/starcoder",
148
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
149
- end_of_text_token="<|endoftext|>",
150
- prefix_token="<|endoftext|>",
151
- ),
152
- TokenizerConfig(
153
- name="google/t5-11b",
154
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
155
- end_of_text_token="</s>",
156
- prefix_token="",
157
- ),
158
- TokenizerConfig(
159
- name="google/flan-t5-xxl",
160
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
161
- end_of_text_token="</s>",
162
- prefix_token="",
163
- ),
164
- TokenizerConfig(
165
- name="google/ul2",
166
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
167
- end_of_text_token="</s>",
168
- prefix_token="",
169
- ),
170
- TokenizerConfig(
171
- name="google/mt5-base",
172
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
173
- end_of_text_token="</s>",
174
- prefix_token="",
175
- ),
176
- TokenizerConfig(
177
- name="facebook/opt-66b",
178
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
179
- end_of_text_token="</s>",
180
- prefix_token="</s>",
181
- ),
182
- TokenizerConfig(
183
- name="openai/cl100k_base",
184
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"),
185
- end_of_text_token="<|endoftext|>",
186
- prefix_token="<|endoftext|>",
187
- ),
188
- TokenizerConfig(
189
- name="TsinghuaKEG/ice",
190
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.ice_tokenizer.ICETokenizer"),
191
- end_of_text_token="</s>",
192
- prefix_token="",
193
- ),
194
- TokenizerConfig(
195
- name="Yandex/yalm",
196
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.yalm_tokenizer.YaLMTokenizer"),
197
- end_of_text_token="</s>",
198
- prefix_token="</s>",
199
- ),
200
- TokenizerConfig(
201
- name="lightningai/lit-gpt",
202
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.lit_gpt_tokenizer.LitGPTTokenizer", args={}),
203
- end_of_text_token="<|endoftext|>",
204
- prefix_token="<|endoftext|>",
205
- ),
206
- TokenizerConfig(
207
- name="HuggingFaceM4/idefics-9b",
208
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
209
- end_of_text_token="</s>",
210
- prefix_token="<s>",
211
- ),
212
- TokenizerConfig(
213
- name="HuggingFaceM4/idefics-9b-instruct",
214
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
215
- end_of_text_token="</s>",
216
- prefix_token="<s>",
217
- ),
218
- TokenizerConfig(
219
- name="HuggingFaceM4/idefics-80b",
220
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
221
- end_of_text_token="</s>",
222
- prefix_token="<s>",
223
- ),
224
- TokenizerConfig(
225
- name="HuggingFaceM4/idefics-80b-instruct",
226
- tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
227
- end_of_text_token="</s>",
228
- prefix_token="<s>",
229
- ),
230
- ]
231
-
232
-
233
- _BUILT_IN_MODEL_DEPLOYMENTS = [
234
- ModelDeployment(
235
- name="neurips/local",
236
- client_spec=ClientSpec(class_name="helm.proxy.clients.http_model_client.HTTPModelClient"),
237
- tokenizer_name="neurips/local",
238
- window_service_spec=WindowServiceSpec(
239
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
240
- ),
241
- max_sequence_length=2048,
242
- ),
243
- ModelDeployment(
244
- name="ai21/j1-jumbo",
245
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
246
- tokenizer_name="ai21/j1",
247
- window_service_spec=WindowServiceSpec(
248
- class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
249
- ),
250
- max_sequence_length=2047,
251
- ),
252
- ModelDeployment(
253
- name="ai21/j1-grande",
254
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
255
- tokenizer_name="ai21/j1",
256
- window_service_spec=WindowServiceSpec(
257
- class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
258
- ),
259
- max_sequence_length=2047,
260
- ),
261
- ModelDeployment(
262
- name="ai21/j1-grande-v2-beta",
263
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
264
- tokenizer_name="ai21/j1",
265
- window_service_spec=WindowServiceSpec(
266
- class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
267
- ),
268
- max_sequence_length=2047,
269
- ),
270
- ModelDeployment(
271
- name="ai21/j1-large",
272
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
273
- tokenizer_name="ai21/j1",
274
- window_service_spec=WindowServiceSpec(
275
- class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
276
- ),
277
- max_sequence_length=2047,
278
- ),
279
- ModelDeployment(
280
- name="ai21/j2-jumbo",
281
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
282
- tokenizer_name="ai21/j1",
283
- window_service_spec=WindowServiceSpec(
284
- class_name="helm.benchmark.window_services.wider_ai21_window_service.AI21Jurassic2JumboWindowService",
285
- args={},
286
- ),
287
- max_sequence_length=6000,
288
- ),
289
- ModelDeployment(
290
- name="ai21/j2-grande",
291
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
292
- tokenizer_name="ai21/j1",
293
- window_service_spec=WindowServiceSpec(
294
- class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
295
- ),
296
- max_sequence_length=2047,
297
- ),
298
- ModelDeployment(
299
- name="ai21/j2-large",
300
- client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
301
- tokenizer_name="ai21/j1",
302
- window_service_spec=WindowServiceSpec(
303
- class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
304
- ),
305
- max_sequence_length=2047,
306
- ),
307
- ModelDeployment(
308
- name="AlephAlpha/luminous-base",
309
- client_spec=ClientSpec(class_name="helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"),
310
- tokenizer_name="AlephAlpha/luminous-base",
311
- window_service_spec=WindowServiceSpec(
312
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
313
- ),
314
- max_sequence_length=2048,
315
- ),
316
- ModelDeployment(
317
- name="AlephAlpha/luminous-extended",
318
- client_spec=ClientSpec(class_name="helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"),
319
- tokenizer_name="AlephAlpha/luminous-extended",
320
- window_service_spec=WindowServiceSpec(
321
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
322
- ),
323
- max_sequence_length=2048,
324
- ),
325
- ModelDeployment(
326
- name="AlephAlpha/luminous-supreme",
327
- client_spec=ClientSpec(class_name="helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"),
328
- tokenizer_name="AlephAlpha/luminous-supreme",
329
- window_service_spec=WindowServiceSpec(
330
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
331
- ),
332
- max_sequence_length=2048,
333
- ),
334
- ModelDeployment(
335
- name="anthropic/stanford-online-all-v4-s3",
336
- client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicLegacyClient"),
337
- tokenizer_name="huggingface/gpt2",
338
- window_service_spec=WindowServiceSpec(
339
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
340
- ),
341
- max_sequence_length=8192,
342
- ),
343
- ModelDeployment(
344
- name="anthropic/claude-2.0",
345
- client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
346
- tokenizer_name="anthropic/claude",
347
- window_service_spec=WindowServiceSpec(
348
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
349
- ),
350
- max_sequence_length=8000,
351
- max_sequence_and_generated_tokens_length=9016,
352
- ),
353
- ModelDeployment(
354
- name="anthropic/claude-2.1",
355
- client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
356
- tokenizer_name="anthropic/claude",
357
- window_service_spec=WindowServiceSpec(
358
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
359
- ),
360
- max_sequence_length=8000,
361
- max_sequence_and_generated_tokens_length=9016,
362
- ),
363
- ModelDeployment(
364
- name="anthropic/claude-v1.3",
365
- client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
366
- tokenizer_name="anthropic/claude",
367
- window_service_spec=WindowServiceSpec(
368
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
369
- ),
370
- max_sequence_length=8000,
371
- max_sequence_and_generated_tokens_length=9016,
372
- ),
373
- ModelDeployment(
374
- name="anthropic/claude-instant-v1",
375
- client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
376
- tokenizer_name="anthropic/claude",
377
- window_service_spec=WindowServiceSpec(
378
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
379
- ),
380
- max_sequence_length=8000,
381
- max_sequence_and_generated_tokens_length=9016,
382
- ),
383
- ModelDeployment(
384
- name="together/bloom",
385
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
386
- tokenizer_name="bigscience/bloom",
387
- window_service_spec=WindowServiceSpec(
388
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
389
- ),
390
- max_sequence_length=2048,
391
- max_request_length=2049,
392
- ),
393
- ModelDeployment(
394
- name="together/t0pp",
395
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
396
- tokenizer_name="bigscience/T0pp",
397
- window_service_spec=WindowServiceSpec(
398
- class_name="helm.benchmark.window_services.t0pp_window_service.T0ppWindowService"
399
- ),
400
- max_sequence_length=1024,
401
- ),
402
- ModelDeployment(
403
- name="cohere/xlarge-20220609",
404
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
405
- tokenizer_name="cohere/cohere",
406
- window_service_spec=WindowServiceSpec(
407
- class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
408
- ),
409
- max_sequence_length=2047,
410
- max_request_length=2048,
411
- ),
412
- ModelDeployment(
413
- name="cohere/xlarge-20221108",
414
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
415
- tokenizer_name="cohere/cohere",
416
- window_service_spec=WindowServiceSpec(
417
- class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
418
- ),
419
- max_sequence_length=2047,
420
- max_request_length=2048,
421
- ),
422
- ModelDeployment(
423
- name="cohere/large-20220720",
424
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
425
- tokenizer_name="cohere/cohere",
426
- window_service_spec=WindowServiceSpec(
427
- class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
428
- ),
429
- max_sequence_length=2047,
430
- max_request_length=2048,
431
- ),
432
- ModelDeployment(
433
- name="cohere/medium-20220720",
434
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
435
- tokenizer_name="cohere/cohere",
436
- window_service_spec=WindowServiceSpec(
437
- class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
438
- ),
439
- max_sequence_length=2047,
440
- max_request_length=2048,
441
- ),
442
- ModelDeployment(
443
- name="cohere/medium-20221108",
444
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
445
- tokenizer_name="cohere/cohere",
446
- window_service_spec=WindowServiceSpec(
447
- class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
448
- ),
449
- max_sequence_length=2047,
450
- max_request_length=2048,
451
- ),
452
- ModelDeployment(
453
- name="cohere/small-20220720",
454
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
455
- tokenizer_name="cohere/cohere",
456
- window_service_spec=WindowServiceSpec(
457
- class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
458
- ),
459
- max_sequence_length=2047,
460
- max_request_length=2048,
461
- ),
462
- ModelDeployment(
463
- name="cohere/command-medium-beta",
464
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
465
- tokenizer_name="cohere/cohere",
466
- window_service_spec=WindowServiceSpec(
467
- class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
468
- ),
469
- max_sequence_length=2019,
470
- max_request_length=2020,
471
- ),
472
- ModelDeployment(
473
- name="cohere/command-xlarge-beta",
474
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
475
- tokenizer_name="cohere/cohere",
476
- window_service_spec=WindowServiceSpec(
477
- class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
478
- ),
479
- max_sequence_length=2019,
480
- max_request_length=2020,
481
- ),
482
- ModelDeployment(
483
- name="cohere/command",
484
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
485
- tokenizer_name="cohere/cohere",
486
- window_service_spec=WindowServiceSpec(
487
- class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
488
- ),
489
- max_sequence_length=2019,
490
- max_request_length=2020,
491
- ),
492
- ModelDeployment(
493
- name="cohere/command-light",
494
- client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
495
- tokenizer_name="cohere/cohere",
496
- window_service_spec=WindowServiceSpec(
497
- class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
498
- ),
499
- max_sequence_length=2019,
500
- max_request_length=2020,
501
- ),
502
- ModelDeployment(
503
- name="together/gpt-j-6b",
504
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
505
- tokenizer_name="EleutherAI/gpt-j-6B",
506
- window_service_spec=WindowServiceSpec(
507
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
508
- ),
509
- max_sequence_length=2048,
510
- max_request_length=2049,
511
- ),
512
- ModelDeployment(
513
- name="together/gpt-neox-20b",
514
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
515
- tokenizer_name="EleutherAI/gpt-neox-20b",
516
- window_service_spec=WindowServiceSpec(
517
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
518
- ),
519
- max_sequence_length=2048,
520
- max_request_length=2049,
521
- ),
522
- ModelDeployment(
523
- name="together/pythia-1b-v0",
524
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
525
- tokenizer_name="EleutherAI/gpt-neox-20b",
526
- window_service_spec=WindowServiceSpec(
527
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
528
- ),
529
- max_sequence_length=2048,
530
- max_request_length=2049,
531
- ),
532
- ModelDeployment(
533
- name="together/pythia-2.8b-v0",
534
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
535
- tokenizer_name="EleutherAI/gpt-neox-20b",
536
- window_service_spec=WindowServiceSpec(
537
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
538
- ),
539
- max_sequence_length=2048,
540
- max_request_length=2049,
541
- ),
542
- ModelDeployment(
543
- name="together/pythia-6.9b",
544
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
545
- tokenizer_name="EleutherAI/gpt-neox-20b",
546
- window_service_spec=WindowServiceSpec(
547
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
548
- ),
549
- max_sequence_length=2048,
550
- max_request_length=2049,
551
- ),
552
- ModelDeployment(
553
- name="together/pythia-12b-v0",
554
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
555
- tokenizer_name="EleutherAI/gpt-neox-20b",
556
- window_service_spec=WindowServiceSpec(
557
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
558
- ),
559
- max_sequence_length=2048,
560
- max_request_length=2049,
561
- ),
562
- ModelDeployment(
563
- name="together/llama-7b",
564
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
565
- tokenizer_name="hf-internal-testing/llama-tokenizer",
566
- window_service_spec=WindowServiceSpec(
567
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
568
- ),
569
- max_sequence_length=2047,
570
- ),
571
- ModelDeployment(
572
- name="together/llama-13b",
573
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
574
- tokenizer_name="hf-internal-testing/llama-tokenizer",
575
- window_service_spec=WindowServiceSpec(
576
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
577
- ),
578
- max_sequence_length=2047,
579
- ),
580
- ModelDeployment(
581
- name="together/llama-30b",
582
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
583
- tokenizer_name="hf-internal-testing/llama-tokenizer",
584
- window_service_spec=WindowServiceSpec(
585
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
586
- ),
587
- max_sequence_length=2047,
588
- ),
589
- ModelDeployment(
590
- name="together/llama-65b",
591
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
592
- tokenizer_name="hf-internal-testing/llama-tokenizer",
593
- window_service_spec=WindowServiceSpec(
594
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
595
- ),
596
- max_sequence_length=2047,
597
- ),
598
- ModelDeployment(
599
- name="together/llama-2-7b",
600
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
601
- tokenizer_name="meta-llama/Llama-2-7b-hf",
602
- window_service_spec=WindowServiceSpec(
603
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
604
- ),
605
- max_sequence_length=4094,
606
- ),
607
- ModelDeployment(
608
- name="together/llama-2-13b",
609
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
610
- tokenizer_name="meta-llama/Llama-2-7b-hf",
611
- window_service_spec=WindowServiceSpec(
612
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
613
- ),
614
- max_sequence_length=4094,
615
- ),
616
- ModelDeployment(
617
- name="together/llama-2-70b",
618
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
619
- tokenizer_name="meta-llama/Llama-2-7b-hf",
620
- window_service_spec=WindowServiceSpec(
621
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
622
- ),
623
- max_sequence_length=4094,
624
- ),
625
- ModelDeployment(
626
- name="together/alpaca-7b",
627
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
628
- tokenizer_name="hf-internal-testing/llama-tokenizer",
629
- window_service_spec=WindowServiceSpec(
630
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
631
- ),
632
- max_sequence_length=2048,
633
- ),
634
- ModelDeployment(
635
- name="together/vicuna-7b-v1.3",
636
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
637
- tokenizer_name="hf-internal-testing/llama-tokenizer",
638
- window_service_spec=WindowServiceSpec(
639
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
640
- ),
641
- max_sequence_length=2048,
642
- ),
643
- ModelDeployment(
644
- name="together/vicuna-13b-v1.3",
645
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
646
- tokenizer_name="hf-internal-testing/llama-tokenizer",
647
- window_service_spec=WindowServiceSpec(
648
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
649
- ),
650
- max_sequence_length=2048,
651
- ),
652
- ModelDeployment(
653
- name="together/mistral-7b-v0.1",
654
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
655
- tokenizer_name="mistralai/Mistral-7B-v0.1",
656
- window_service_spec=WindowServiceSpec(
657
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
658
- ),
659
- max_sequence_length=4095,
660
- ),
661
- ModelDeployment(
662
- name="together/mpt-7b",
663
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
664
- tokenizer_name="EleutherAI/gpt-neox-20b",
665
- window_service_spec=WindowServiceSpec(
666
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
667
- ),
668
- max_sequence_length=2048,
669
- max_request_length=2049,
670
- ),
671
- ModelDeployment(
672
- name="together/mpt-instruct-7b",
673
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
674
- tokenizer_name="EleutherAI/gpt-neox-20b",
675
- window_service_spec=WindowServiceSpec(
676
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
677
- ),
678
- max_sequence_length=2048,
679
- max_request_length=2049,
680
- ),
681
- ModelDeployment(
682
- name="together/mpt-30b",
683
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
684
- tokenizer_name="EleutherAI/gpt-neox-20b",
685
- window_service_spec=WindowServiceSpec(
686
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
687
- ),
688
- max_sequence_length=2048,
689
- max_request_length=2049,
690
- ),
691
- ModelDeployment(
692
- name="together/mpt-instruct-30b",
693
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
694
- tokenizer_name="EleutherAI/gpt-neox-20b",
695
- window_service_spec=WindowServiceSpec(
696
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
697
- ),
698
- max_sequence_length=2048,
699
- max_request_length=2049,
700
- ),
701
- ModelDeployment(
702
- name="together/falcon-7b",
703
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
704
- tokenizer_name="tiiuae/falcon-7b",
705
- window_service_spec=WindowServiceSpec(
706
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
707
- ),
708
- max_sequence_length=2047,
709
- ),
710
- ModelDeployment(
711
- name="together/falcon-7b-instruct",
712
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
713
- tokenizer_name="tiiuae/falcon-7b",
714
- window_service_spec=WindowServiceSpec(
715
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
716
- ),
717
- max_sequence_length=2047,
718
- ),
719
- ModelDeployment(
720
- name="together/falcon-40b",
721
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
722
- tokenizer_name="tiiuae/falcon-7b",
723
- window_service_spec=WindowServiceSpec(
724
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
725
- ),
726
- max_sequence_length=2047,
727
- ),
728
- ModelDeployment(
729
- name="together/falcon-40b-instruct",
730
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
731
- tokenizer_name="tiiuae/falcon-7b",
732
- window_service_spec=WindowServiceSpec(
733
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
734
- ),
735
- max_sequence_length=2047,
736
- ),
737
- ModelDeployment(
738
- name="gooseai/gpt-neo-20b",
739
- client_spec=ClientSpec(class_name="helm.proxy.clients.goose_ai_client.GooseAIClient"),
740
- tokenizer_name="EleutherAI/gpt-neox-20b",
741
- window_service_spec=WindowServiceSpec(
742
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
743
- ),
744
- max_sequence_length=2048,
745
- max_request_length=2049,
746
- ),
747
- ModelDeployment(
748
- name="gooseai/gpt-j-6b",
749
- client_spec=ClientSpec(class_name="helm.proxy.clients.goose_ai_client.GooseAIClient"),
750
- tokenizer_name="EleutherAI/gpt-j-6B",
751
- window_service_spec=WindowServiceSpec(
752
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
753
- ),
754
- max_sequence_length=2048,
755
- max_request_length=2049,
756
- ),
757
- ModelDeployment(
758
- name="huggingface/gpt2",
759
- client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
760
- tokenizer_name="huggingface/gpt2",
761
- window_service_spec=WindowServiceSpec(
762
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
763
- ),
764
- max_sequence_length=1024,
765
- max_request_length=1025,
766
- ),
767
- ModelDeployment(
768
- name="huggingface/gpt-j-6b",
769
- client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
770
- tokenizer_name="EleutherAI/gpt-j-6B",
771
- window_service_spec=WindowServiceSpec(
772
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
773
- ),
774
- max_sequence_length=2048,
775
- max_request_length=2049,
776
- ),
777
- ModelDeployment(
778
- name="huggingface/santacoder",
779
- client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
780
- tokenizer_name="bigcode/santacoder",
781
- window_service_spec=WindowServiceSpec(
782
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
783
- ),
784
- max_sequence_length=2048,
785
- ),
786
- ModelDeployment(
787
- name="huggingface/starcoder",
788
- client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
789
- tokenizer_name="bigcode/starcoder",
790
- window_service_spec=WindowServiceSpec(
791
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
792
- ),
793
- max_sequence_length=8192,
794
- ),
795
- ModelDeployment(
796
- name="together/t5-11b",
797
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
798
- tokenizer_name="google/t5-11b",
799
- window_service_spec=WindowServiceSpec(
800
- class_name="helm.benchmark.window_services.t511b_window_service.T511bWindowService"
801
- ),
802
- max_sequence_length=511,
803
- ),
804
- ModelDeployment(
805
- name="together/flan-t5-xxl",
806
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
807
- tokenizer_name="google/flan-t5-xxl",
808
- window_service_spec=WindowServiceSpec(
809
- class_name="helm.benchmark.window_services.flan_t5_window_service.FlanT5WindowService"
810
- ),
811
- max_sequence_length=511,
812
- ),
813
- ModelDeployment(
814
- name="together/ul2",
815
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
816
- tokenizer_name="google/ul2",
817
- window_service_spec=WindowServiceSpec(
818
- class_name="helm.benchmark.window_services.ul2_window_service.UL2WindowService"
819
- ),
820
- max_sequence_length=511,
821
- ),
822
- ModelDeployment(
823
- name="google/text-bison@001",
824
- client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
825
- tokenizer_name="google/mt5-base",
826
- window_service_spec=WindowServiceSpec(
827
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
828
- ),
829
- max_sequence_length=6000,
830
- max_sequence_and_generated_tokens_length=7000,
831
- ),
832
- ModelDeployment(
833
- name="google/text-bison-32k",
834
- client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
835
- tokenizer_name="google/mt5-base",
836
- window_service_spec=WindowServiceSpec(
837
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
838
- ),
839
- max_sequence_length=32000,
840
- max_sequence_and_generated_tokens_length=32000,
841
- ),
842
- ModelDeployment(
843
- name="google/text-unicorn@001",
844
- client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
845
- tokenizer_name="google/mt5-base",
846
- window_service_spec=WindowServiceSpec(
847
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
848
- ),
849
- max_sequence_length=6000,
850
- max_sequence_and_generated_tokens_length=7000,
851
- ),
852
- ModelDeployment(
853
- name="google/code-bison@001",
854
- client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
855
- tokenizer_name="google/mt5-base",
856
- window_service_spec=WindowServiceSpec(
857
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
858
- ),
859
- max_sequence_length=6000,
860
- max_sequence_and_generated_tokens_length=7000,
861
- ),
862
- ModelDeployment(
863
- name="google/code-bison-32k",
864
- client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
865
- tokenizer_name="google/mt5-base",
866
- window_service_spec=WindowServiceSpec(
867
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
868
- ),
869
- max_sequence_length=32000,
870
- max_sequence_and_generated_tokens_length=32000,
871
- ),
872
- ModelDeployment(
873
- name="together/h3-2.7b",
874
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
875
- tokenizer_name="huggingface/gpt2",
876
- window_service_spec=WindowServiceSpec(
877
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
878
- ),
879
- max_sequence_length=1024,
880
- max_request_length=1025,
881
- ),
882
- ModelDeployment(
883
- name="together/opt-175b",
884
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
885
- tokenizer_name="facebook/opt-66b",
886
- window_service_spec=WindowServiceSpec(
887
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
888
- ),
889
- max_sequence_length=2048,
890
- max_request_length=2049,
891
- ),
892
- ModelDeployment(
893
- name="together/opt-66b",
894
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
895
- tokenizer_name="facebook/opt-66b",
896
- window_service_spec=WindowServiceSpec(
897
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
898
- ),
899
- max_sequence_length=2048,
900
- max_request_length=2049,
901
- ),
902
- ModelDeployment(
903
- name="together/opt-6.7b",
904
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
905
- tokenizer_name="facebook/opt-66b",
906
- window_service_spec=WindowServiceSpec(
907
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
908
- ),
909
- max_sequence_length=2048,
910
- max_request_length=2049,
911
- ),
912
- ModelDeployment(
913
- name="together/opt-1.3b",
914
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
915
- tokenizer_name="facebook/opt-66b",
916
- window_service_spec=WindowServiceSpec(
917
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
918
- ),
919
- max_sequence_length=2048,
920
- max_request_length=2049,
921
- ),
922
- ModelDeployment(
923
- name="microsoft/TNLGv2_530B",
924
- client_spec=ClientSpec(class_name="helm.proxy.clients.microsoft_client.MicrosoftClient"),
925
- tokenizer_name="microsoft/gpt2",
926
- window_service_spec=WindowServiceSpec(
927
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
928
- ),
929
- max_sequence_length=2047,
930
- max_request_length=2048,
931
- ),
932
- ModelDeployment(
933
- name="microsoft/TNLGv2_7B",
934
- client_spec=ClientSpec(class_name="helm.proxy.clients.microsoft_client.MicrosoftClient"),
935
- tokenizer_name="microsoft/gpt2",
936
- window_service_spec=WindowServiceSpec(
937
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
938
- ),
939
- max_sequence_length=2047,
940
- max_request_length=2048,
941
- ),
942
- ModelDeployment(
943
- name="openai/davinci",
944
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
945
- tokenizer_name="huggingface/gpt2",
946
- window_service_spec=WindowServiceSpec(
947
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
948
- ),
949
- max_sequence_length=2048,
950
- max_request_length=2049,
951
- ),
952
- ModelDeployment(
953
- name="openai/curie",
954
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
955
- tokenizer_name="huggingface/gpt2",
956
- window_service_spec=WindowServiceSpec(
957
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
958
- ),
959
- max_sequence_length=2048,
960
- max_request_length=2049,
961
- ),
962
- ModelDeployment(
963
- name="openai/babbage",
964
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
965
- tokenizer_name="huggingface/gpt2",
966
- window_service_spec=WindowServiceSpec(
967
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
968
- ),
969
- max_sequence_length=2048,
970
- max_request_length=2049,
971
- ),
972
- ModelDeployment(
973
- name="openai/ada",
974
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
975
- tokenizer_name="huggingface/gpt2",
976
- window_service_spec=WindowServiceSpec(
977
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
978
- ),
979
- max_sequence_length=2048,
980
- max_request_length=2049,
981
- ),
982
- ModelDeployment(
983
- name="openai/text-davinci-003",
984
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
985
- tokenizer_name="huggingface/gpt2",
986
- window_service_spec=WindowServiceSpec(
987
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
988
- ),
989
- max_sequence_length=4000,
990
- max_request_length=4001,
991
- ),
992
- ModelDeployment(
993
- name="openai/text-davinci-002",
994
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
995
- tokenizer_name="huggingface/gpt2",
996
- window_service_spec=WindowServiceSpec(
997
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
998
- ),
999
- max_sequence_length=4000,
1000
- max_request_length=4001,
1001
- ),
1002
- ModelDeployment(
1003
- name="openai/text-davinci-001",
1004
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1005
- tokenizer_name="huggingface/gpt2",
1006
- window_service_spec=WindowServiceSpec(
1007
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1008
- ),
1009
- max_sequence_length=2048,
1010
- max_request_length=2049,
1011
- ),
1012
- ModelDeployment(
1013
- name="openai/text-curie-001",
1014
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1015
- tokenizer_name="huggingface/gpt2",
1016
- window_service_spec=WindowServiceSpec(
1017
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1018
- ),
1019
- max_sequence_length=2048,
1020
- max_request_length=2049,
1021
- ),
1022
- ModelDeployment(
1023
- name="openai/text-babbage-001",
1024
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1025
- tokenizer_name="huggingface/gpt2",
1026
- window_service_spec=WindowServiceSpec(
1027
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1028
- ),
1029
- max_sequence_length=2048,
1030
- max_request_length=2049,
1031
- ),
1032
- ModelDeployment(
1033
- name="openai/text-ada-001",
1034
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1035
- tokenizer_name="huggingface/gpt2",
1036
- window_service_spec=WindowServiceSpec(
1037
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1038
- ),
1039
- max_sequence_length=2048,
1040
- max_request_length=2049,
1041
- ),
1042
- ModelDeployment(
1043
- name="openai/code-davinci-002",
1044
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1045
- tokenizer_name="huggingface/gpt2",
1046
- window_service_spec=WindowServiceSpec(
1047
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1048
- ),
1049
- max_sequence_length=4000,
1050
- max_request_length=4001,
1051
- ),
1052
- ModelDeployment(
1053
- name="openai/code-davinci-001",
1054
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1055
- tokenizer_name="huggingface/gpt2",
1056
- window_service_spec=WindowServiceSpec(
1057
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1058
- ),
1059
- max_sequence_length=2048,
1060
- max_request_length=2049,
1061
- ),
1062
- ModelDeployment(
1063
- name="openai/code-cushman-001",
1064
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1065
- tokenizer_name="huggingface/gpt2",
1066
- window_service_spec=WindowServiceSpec(
1067
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1068
- ),
1069
- max_sequence_length=2048,
1070
- max_request_length=2049,
1071
- ),
1072
- ModelDeployment(
1073
- name="openai/gpt-4-1106-preview",
1074
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1075
- tokenizer_name="openai/cl100k_base",
1076
- window_service_spec=WindowServiceSpec(
1077
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1078
- ),
1079
- max_sequence_length=128000,
1080
- max_request_length=128001,
1081
- ),
1082
- ModelDeployment(
1083
- name="openai/gpt-4-0314",
1084
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1085
- tokenizer_name="openai/cl100k_base",
1086
- window_service_spec=WindowServiceSpec(
1087
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1088
- ),
1089
- max_sequence_length=8192,
1090
- max_request_length=8193,
1091
- ),
1092
- ModelDeployment(
1093
- name="openai/gpt-4-32k-0314",
1094
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1095
- tokenizer_name="openai/cl100k_base",
1096
- window_service_spec=WindowServiceSpec(
1097
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1098
- ),
1099
- max_sequence_length=32768,
1100
- max_request_length=32769,
1101
- ),
1102
- ModelDeployment(
1103
- name="openai/gpt-4-0613",
1104
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1105
- tokenizer_name="openai/cl100k_base",
1106
- window_service_spec=WindowServiceSpec(
1107
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1108
- ),
1109
- max_sequence_length=8192,
1110
- max_request_length=8193,
1111
- ),
1112
- ModelDeployment(
1113
- name="openai/gpt-4-32k-0613",
1114
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1115
- tokenizer_name="openai/cl100k_base",
1116
- window_service_spec=WindowServiceSpec(
1117
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1118
- ),
1119
- max_sequence_length=32768,
1120
- max_request_length=32769,
1121
- ),
1122
- ModelDeployment(
1123
- name="openai/gpt-3.5-turbo-0301",
1124
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1125
- tokenizer_name="openai/cl100k_base",
1126
- window_service_spec=WindowServiceSpec(
1127
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1128
- ),
1129
- max_sequence_length=4000,
1130
- max_request_length=4001,
1131
- ),
1132
- ModelDeployment(
1133
- name="openai/gpt-3.5-turbo-0613",
1134
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1135
- tokenizer_name="openai/cl100k_base",
1136
- window_service_spec=WindowServiceSpec(
1137
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1138
- ),
1139
- max_sequence_length=4000,
1140
- max_request_length=4001,
1141
- ),
1142
- ModelDeployment(
1143
- name="openai/gpt-3.5-turbo-16k-0613",
1144
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1145
- tokenizer_name="openai/cl100k_base",
1146
- window_service_spec=WindowServiceSpec(
1147
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1148
- ),
1149
- max_sequence_length=16000,
1150
- max_request_length=16001,
1151
- ),
1152
- ModelDeployment(
1153
- name="openai/text-similarity-davinci-001",
1154
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1155
- tokenizer_name="huggingface/gpt2",
1156
- window_service_spec=WindowServiceSpec(
1157
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1158
- ),
1159
- max_sequence_length=2048,
1160
- max_request_length=2049,
1161
- ),
1162
- ModelDeployment(
1163
- name="openai/text-similarity-curie-001",
1164
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1165
- tokenizer_name="huggingface/gpt2",
1166
- window_service_spec=WindowServiceSpec(
1167
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1168
- ),
1169
- max_sequence_length=2048,
1170
- max_request_length=2049,
1171
- ),
1172
- ModelDeployment(
1173
- name="openai/text-similarity-babbage-001",
1174
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1175
- tokenizer_name="huggingface/gpt2",
1176
- window_service_spec=WindowServiceSpec(
1177
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1178
- ),
1179
- max_sequence_length=2048,
1180
- max_request_length=2049,
1181
- ),
1182
- ModelDeployment(
1183
- name="openai/text-similarity-ada-001",
1184
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1185
- tokenizer_name="huggingface/gpt2",
1186
- window_service_spec=WindowServiceSpec(
1187
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1188
- ),
1189
- max_sequence_length=2048,
1190
- max_request_length=2049,
1191
- ),
1192
- ModelDeployment(
1193
- name="openai/text-embedding-ada-002",
1194
- client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
1195
- tokenizer_name="huggingface/gpt2",
1196
- window_service_spec=WindowServiceSpec(
1197
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1198
- ),
1199
- max_sequence_length=2048,
1200
- max_request_length=2049,
1201
- ),
1202
- ModelDeployment(
1203
- name="together/gpt-jt-6b-v1",
1204
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1205
- tokenizer_name="EleutherAI/gpt-j-6B",
1206
- window_service_spec=WindowServiceSpec(
1207
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1208
- ),
1209
- max_sequence_length=2048,
1210
- max_request_length=2049,
1211
- ),
1212
- ModelDeployment(
1213
- name="together/gpt-neoxt-chat-base-20b",
1214
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1215
- tokenizer_name="EleutherAI/gpt-neox-20b",
1216
- window_service_spec=WindowServiceSpec(
1217
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1218
- ),
1219
- max_sequence_length=2048,
1220
- max_request_length=2049,
1221
- ),
1222
- ModelDeployment(
1223
- name="together/redpajama-incite-base-3b-v1",
1224
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1225
- tokenizer_name="EleutherAI/gpt-neox-20b",
1226
- window_service_spec=WindowServiceSpec(
1227
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1228
- ),
1229
- max_sequence_length=2048,
1230
- max_request_length=2049,
1231
- ),
1232
- ModelDeployment(
1233
- name="together/redpajama-incite-instruct-3b-v1",
1234
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1235
- tokenizer_name="EleutherAI/gpt-neox-20b",
1236
- window_service_spec=WindowServiceSpec(
1237
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1238
- ),
1239
- max_sequence_length=2048,
1240
- max_request_length=2049,
1241
- ),
1242
- ModelDeployment(
1243
- name="together/redpajama-incite-base-7b",
1244
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1245
- tokenizer_name="EleutherAI/gpt-neox-20b",
1246
- window_service_spec=WindowServiceSpec(
1247
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1248
- ),
1249
- max_sequence_length=2048,
1250
- max_request_length=2049,
1251
- ),
1252
- ModelDeployment(
1253
- name="together/redpajama-incite-instruct-7b",
1254
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1255
- tokenizer_name="EleutherAI/gpt-neox-20b",
1256
- window_service_spec=WindowServiceSpec(
1257
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1258
- ),
1259
- max_sequence_length=2048,
1260
- max_request_length=2049,
1261
- ),
1262
- ModelDeployment(
1263
- name="together/glm",
1264
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1265
- tokenizer_name="TsinghuaKEG/ice",
1266
- window_service_spec=WindowServiceSpec(
1267
- class_name="helm.benchmark.window_services.ice_window_service.ICEWindowService"
1268
- ),
1269
- max_sequence_length=2048,
1270
- max_request_length=2049,
1271
- ),
1272
- ModelDeployment(
1273
- name="writer/palmyra-base",
1274
- client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
1275
- tokenizer_name="writer/gpt2",
1276
- window_service_spec=WindowServiceSpec(
1277
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1278
- ),
1279
- max_sequence_length=2048,
1280
- max_sequence_and_generated_tokens_length=2048,
1281
- ),
1282
- ModelDeployment(
1283
- name="writer/palmyra-large",
1284
- client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
1285
- tokenizer_name="writer/gpt2",
1286
- window_service_spec=WindowServiceSpec(
1287
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1288
- ),
1289
- max_sequence_length=2048,
1290
- max_sequence_and_generated_tokens_length=2048,
1291
- ),
1292
- ModelDeployment(
1293
- name="writer/palmyra-instruct-30",
1294
- client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
1295
- tokenizer_name="writer/gpt2",
1296
- window_service_spec=WindowServiceSpec(
1297
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1298
- ),
1299
- max_sequence_length=2048,
1300
- max_sequence_and_generated_tokens_length=2048,
1301
- ),
1302
- ModelDeployment(
1303
- name="writer/palmyra-e",
1304
- client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
1305
- tokenizer_name="writer/gpt2",
1306
- window_service_spec=WindowServiceSpec(
1307
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1308
- ),
1309
- max_sequence_length=2048,
1310
- max_sequence_and_generated_tokens_length=2048,
1311
- ),
1312
- ModelDeployment(
1313
- name="writer/silk-road",
1314
- client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
1315
- tokenizer_name="writer/gpt2",
1316
- window_service_spec=WindowServiceSpec(
1317
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1318
- ),
1319
- max_sequence_length=8192,
1320
- max_sequence_and_generated_tokens_length=8192,
1321
- ),
1322
- ModelDeployment(
1323
- name="writer/palmyra-x",
1324
- client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
1325
- tokenizer_name="writer/gpt2",
1326
- window_service_spec=WindowServiceSpec(
1327
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1328
- ),
1329
- max_sequence_length=8192,
1330
- max_sequence_and_generated_tokens_length=8192,
1331
- ),
1332
- ModelDeployment(
1333
- name="together/yalm",
1334
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1335
- tokenizer_name="Yandex/yalm",
1336
- window_service_spec=WindowServiceSpec(
1337
- class_name="helm.benchmark.window_services.yalm_window_service.YaLMWindowService"
1338
- ),
1339
- max_sequence_length=2048,
1340
- max_request_length=2049,
1341
- ),
1342
- ModelDeployment(
1343
- name="nvidia/megatron-gpt2",
1344
- client_spec=ClientSpec(class_name="helm.proxy.clients.megatron_client.MegatronClient"),
1345
- tokenizer_name="huggingface/gpt2",
1346
- window_service_spec=WindowServiceSpec(
1347
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1348
- ),
1349
- max_sequence_length=1024,
1350
- ),
1351
- ModelDeployment(
1352
- name="together/dolly-v2-3b",
1353
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1354
- tokenizer_name="EleutherAI/gpt-neox-20b",
1355
- window_service_spec=WindowServiceSpec(
1356
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1357
- ),
1358
- max_sequence_length=2048,
1359
- max_request_length=2049,
1360
- ),
1361
- ModelDeployment(
1362
- name="together/dolly-v2-7b",
1363
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1364
- tokenizer_name="EleutherAI/gpt-neox-20b",
1365
- window_service_spec=WindowServiceSpec(
1366
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1367
- ),
1368
- max_sequence_length=2048,
1369
- max_request_length=2049,
1370
- ),
1371
- ModelDeployment(
1372
- name="together/dolly-v2-12b",
1373
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1374
- tokenizer_name="EleutherAI/gpt-neox-20b",
1375
- window_service_spec=WindowServiceSpec(
1376
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1377
- ),
1378
- max_sequence_length=2048,
1379
- max_request_length=2049,
1380
- ),
1381
- ModelDeployment(
1382
- name="together/stablelm-base-alpha-3b",
1383
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1384
- tokenizer_name="EleutherAI/gpt-neox-20b",
1385
- window_service_spec=WindowServiceSpec(
1386
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1387
- ),
1388
- max_sequence_length=4096,
1389
- max_request_length=4097,
1390
- ),
1391
- ModelDeployment(
1392
- name="together/stablelm-base-alpha-7b",
1393
- client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
1394
- tokenizer_name="EleutherAI/gpt-neox-20b",
1395
- window_service_spec=WindowServiceSpec(
1396
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1397
- ),
1398
- max_sequence_length=4096,
1399
- max_request_length=4097,
1400
- ),
1401
- ModelDeployment(
1402
- name="lightningai/lit-gpt",
1403
- client_spec=ClientSpec(class_name="helm.proxy.clients.lit_gpt_client.LitGPTClient", args={}),
1404
- model_name=None,
1405
- tokenizer_name="lightningai/lit-gpt",
1406
- window_service_spec=WindowServiceSpec(
1407
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService", args={}
1408
- ),
1409
- max_sequence_length=2048,
1410
- max_request_length=None,
1411
- max_sequence_and_generated_tokens_length=None,
1412
- ),
1413
- ModelDeployment(
1414
- name="HuggingFaceM4/idefics-9b",
1415
- client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
1416
- tokenizer_name="HuggingFaceM4/idefics-9b",
1417
- window_service_spec=WindowServiceSpec(
1418
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1419
- ),
1420
- max_sequence_length=2048,
1421
- ),
1422
- ModelDeployment(
1423
- name="HuggingFaceM4/idefics-9b-instruct",
1424
- client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
1425
- tokenizer_name="HuggingFaceM4/idefics-9b-instruct",
1426
- window_service_spec=WindowServiceSpec(
1427
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1428
- ),
1429
- max_sequence_length=2048,
1430
- ),
1431
- ModelDeployment(
1432
- name="HuggingFaceM4/idefics-80b",
1433
- client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
1434
- tokenizer_name="HuggingFaceM4/idefics-80b",
1435
- window_service_spec=WindowServiceSpec(
1436
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1437
- ),
1438
- max_sequence_length=2048,
1439
- ),
1440
- ModelDeployment(
1441
- name="HuggingFaceM4/idefics-80b-instruct",
1442
- client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
1443
- tokenizer_name="HuggingFaceM4/idefics-80b-instruct",
1444
- window_service_spec=WindowServiceSpec(
1445
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1446
- ),
1447
- max_sequence_length=2048,
1448
- ),
1449
- ModelDeployment(
1450
- name="simple/model1",
1451
- client_spec=ClientSpec(class_name="helm.proxy.clients.simple_client.SimpleClient"),
1452
- tokenizer_name="simple/model1",
1453
- window_service_spec=WindowServiceSpec(
1454
- class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
1455
- ),
1456
- max_sequence_length=2048,
1457
- ),
1458
- ]
1459
-
1460
-
1461
- _INT_MAX: int = 2**31 - 1
1462
-
1463
-
1464
- def _full_class_name(obj: Any) -> str:
1465
- return f"{obj.__class__.__module__}.{obj.__class__.__name__}"
1466
-
1467
-
1468
- # HACK: This looks like it should be done in a setup_class()
1469
- # for the test below but apparently pytest first check the parametrize
1470
- # before running the setup_class().
1471
- # Therefore ALL_MODEL_DEPLOYMENTS is empty and no test would be run,
1472
- # so we need to do this here.
1473
- register_builtin_configs_from_helm_package()
1474
-
1475
-
1476
- class TestModelProperties:
1477
- @pytest.mark.parametrize("deployment_name", [deployment.name for deployment in _BUILT_IN_MODEL_DEPLOYMENTS])
1478
- def test_models_has_window_service(self, deployment_name: str):
1479
- auto_client = AutoClient(defaultdict(str), "", "")
1480
- auto_tokenizer = AutoTokenizer(defaultdict(str), "", "")
1481
- model_deployments = {
1482
- model_deployment.name: model_deployment for model_deployment in _BUILT_IN_MODEL_DEPLOYMENTS
1483
- }
1484
- tokenizer_configs = {
1485
- tokenizer_config.name: tokenizer_config for tokenizer_config in _BUILT_IN_TOKENIZER_CONFIGS
1486
- }
1487
- with TemporaryDirectory() as tmpdir:
1488
- tokenizer_service = get_tokenizer_service(tmpdir)
1489
- deployment: ModelDeployment = get_model_deployment(deployment_name)
1490
- model: ModelMetadata = get_model_metadata(deployment.model_name or deployment_name)
1491
- # Can't test lit-gpt client because it requires manual dependencies
1492
- if "lit-gpt" in model.name:
1493
- return
1494
-
1495
- # Can't test Llama 2 because it requires Hugging Face credentials
1496
- if "llama-2-" in model.name:
1497
- return
1498
-
1499
- # Can't test Vertex AI because it requires Google credentials
1500
- if "text-bison" in model.name or "text-unicorn" in model.name:
1501
- return
1502
-
1503
- client = auto_client._get_client(deployment_name)
1504
- window_service = WindowServiceFactory.get_window_service(deployment_name, tokenizer_service)
1505
- tokenizer_name = window_service.tokenizer_name
1506
- tokenizer = auto_tokenizer._get_tokenizer(tokenizer_name)
1507
-
1508
- client_class_name = _full_class_name(client)
1509
- tokenizer_class_name = _full_class_name(tokenizer)
1510
- window_service_class_name = _full_class_name(window_service)
1511
-
1512
- prefix_token = window_service.prefix_token
1513
- end_of_text_token = window_service.end_of_text_token
1514
-
1515
- max_sequence_length = window_service.max_sequence_length
1516
- max_request_length = (
1517
- window_service.max_request_length
1518
- if window_service.max_request_length != window_service.max_sequence_length
1519
- else None
1520
- )
1521
- max_sequence_and_generated_tokens_length = (
1522
- window_service.max_sequence_and_generated_tokens_length
1523
- if window_service.max_sequence_and_generated_tokens_length != _INT_MAX
1524
- else None
1525
- )
1526
-
1527
- model_deployment = ModelDeployment(
1528
- name=deployment_name,
1529
- client_spec=ClientSpec(class_name=client_class_name),
1530
- tokenizer_name=tokenizer_name,
1531
- window_service_spec=WindowServiceSpec(class_name=window_service_class_name),
1532
- max_sequence_length=max_sequence_length,
1533
- max_request_length=max_request_length,
1534
- max_sequence_and_generated_tokens_length=max_sequence_and_generated_tokens_length,
1535
- )
1536
- tokenizer_config = TokenizerConfig(
1537
- name=tokenizer_name,
1538
- tokenizer_spec=TokenizerSpec(class_name=tokenizer_class_name),
1539
- end_of_text_token=end_of_text_token,
1540
- prefix_token=prefix_token,
1541
- )
1542
- # NOTE: To generate the _BUILT_IN_MODEL_DEPLOYMENT and _BUILT_IN_TOKENIZER_CONFIGS lists above,
1543
- # print tokenizer_config and model_deployment here.
1544
-
1545
- # Cannot directly compare model_deployment and deployment because they will have different values
1546
- # for deprecated for example as it is not specified in this file.
1547
- # In stead compare all the fields defined here.
1548
- test_model_deployment: ModelDeployment = model_deployments[deployment_name]
1549
- assert model_deployment.name == test_model_deployment.name
1550
- assert model_deployment.client_spec.class_name == test_model_deployment.client_spec.class_name
1551
- assert model_deployment.tokenizer_name == test_model_deployment.tokenizer_name
1552
- if model_deployment.window_service_spec:
1553
- assert test_model_deployment.window_service_spec
1554
- assert (
1555
- model_deployment.window_service_spec.class_name
1556
- == test_model_deployment.window_service_spec.class_name
1557
- )
1558
- else:
1559
- assert not test_model_deployment.window_service_spec
1560
- assert model_deployment.max_sequence_length == test_model_deployment.max_sequence_length
1561
- assert model_deployment.max_request_length == test_model_deployment.max_request_length
1562
- assert (
1563
- model_deployment.max_sequence_and_generated_tokens_length
1564
- == test_model_deployment.max_sequence_and_generated_tokens_length
1565
- )
1566
- # DefaultWindowService overrides the huggingface/gpt2 tokenizer with different special tokens,
1567
- # so there are currently two tokenizers named huggingface/gpt2
1568
- # TODO: Give DefaultWindowService's tokenizer a different name e.g. writer/palmyra
1569
- if tokenizer_name != "huggingface/gpt2":
1570
- assert tokenizer_configs[tokenizer_name] == tokenizer_config