crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (482) hide show
  1. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.0.dist-info}/METADATA +134 -31
  2. crfm_helm-0.5.0.dist-info/RECORD +642 -0
  3. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.0.dist-info}/WHEEL +1 -1
  4. helm/benchmark/adaptation/adapter_spec.py +31 -3
  5. helm/benchmark/adaptation/adapters/adapter.py +2 -2
  6. helm/benchmark/adaptation/adapters/adapter_factory.py +24 -27
  7. helm/benchmark/adaptation/adapters/generation_adapter.py +1 -0
  8. helm/benchmark/adaptation/adapters/in_context_learning_adapter.py +20 -4
  9. helm/benchmark/adaptation/adapters/language_modeling_adapter.py +2 -3
  10. helm/benchmark/adaptation/adapters/multimodal/multiple_choice_joint_multimodal_adapter.py +104 -0
  11. helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py +2 -1
  12. helm/benchmark/adaptation/adapters/test_adapter.py +2 -1
  13. helm/benchmark/adaptation/adapters/test_generation_adapter.py +32 -8
  14. helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py +7 -19
  15. helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py +60 -6
  16. helm/benchmark/adaptation/common_adapter_specs.py +376 -0
  17. helm/benchmark/adaptation/request_state.py +6 -1
  18. helm/benchmark/adaptation/scenario_state.py +6 -2
  19. helm/benchmark/annotation/annotator.py +43 -0
  20. helm/benchmark/annotation/annotator_factory.py +61 -0
  21. helm/benchmark/annotation/image2structure/image_compiler_annotator.py +88 -0
  22. helm/benchmark/annotation/image2structure/latex_compiler_annotator.py +59 -0
  23. helm/benchmark/annotation/image2structure/lilypond_compiler_annotator.py +84 -0
  24. helm/benchmark/annotation/image2structure/webpage_compiler_annotator.py +132 -0
  25. helm/benchmark/annotation/test_annotator_factory.py +26 -0
  26. helm/benchmark/annotation/test_dummy_annotator.py +44 -0
  27. helm/benchmark/annotation_executor.py +124 -0
  28. helm/benchmark/augmentations/data_augmenter.py +0 -2
  29. helm/benchmark/augmentations/gender_perturbation.py +1 -1
  30. helm/benchmark/augmentations/perturbation.py +8 -2
  31. helm/benchmark/augmentations/perturbation_description.py +1 -1
  32. helm/benchmark/augmentations/suffix_perturbation.py +29 -0
  33. helm/benchmark/augmentations/test_perturbation.py +11 -7
  34. helm/benchmark/augmentations/translate_perturbation.py +30 -0
  35. helm/benchmark/config_registry.py +7 -1
  36. helm/benchmark/executor.py +46 -16
  37. helm/benchmark/huggingface_registration.py +20 -7
  38. helm/benchmark/metrics/basic_metrics.py +169 -664
  39. helm/benchmark/metrics/bbq_metrics.py +3 -4
  40. helm/benchmark/metrics/bias_metrics.py +6 -6
  41. helm/benchmark/metrics/classification_metrics.py +11 -8
  42. helm/benchmark/metrics/cleva_accuracy_metrics.py +8 -5
  43. helm/benchmark/metrics/cleva_harms_metrics.py +2 -2
  44. helm/benchmark/metrics/code_metrics_helper.py +0 -2
  45. helm/benchmark/metrics/common_metric_specs.py +167 -0
  46. helm/benchmark/metrics/decodingtrust_fairness_metrics.py +72 -0
  47. helm/benchmark/metrics/decodingtrust_ood_knowledge_metrics.py +66 -0
  48. helm/benchmark/metrics/decodingtrust_privacy_metrics.py +101 -0
  49. helm/benchmark/metrics/decodingtrust_stereotype_bias_metrics.py +202 -0
  50. helm/benchmark/metrics/disinformation_metrics.py +4 -110
  51. helm/benchmark/metrics/dry_run_metrics.py +2 -2
  52. helm/benchmark/metrics/efficiency_metrics.py +206 -0
  53. helm/benchmark/metrics/evaluate_instances_metric.py +59 -0
  54. helm/benchmark/metrics/evaluate_reference_metrics.py +376 -0
  55. helm/benchmark/metrics/image_generation/aesthetics_metrics.py +54 -0
  56. helm/benchmark/metrics/image_generation/aesthetics_scorer.py +66 -0
  57. helm/benchmark/metrics/image_generation/clip_score_metrics.py +73 -0
  58. helm/benchmark/metrics/image_generation/denoised_runtime_metric.py +42 -0
  59. helm/benchmark/metrics/image_generation/detection_metrics.py +57 -0
  60. helm/benchmark/metrics/image_generation/detectors/base_detector.py +8 -0
  61. helm/benchmark/metrics/image_generation/detectors/vitdet.py +178 -0
  62. helm/benchmark/metrics/image_generation/efficiency_metrics.py +41 -0
  63. helm/benchmark/metrics/image_generation/fidelity_metrics.py +168 -0
  64. helm/benchmark/metrics/image_generation/fractal_dimension/__init__.py +0 -0
  65. helm/benchmark/metrics/image_generation/fractal_dimension/fractal_dimension_util.py +63 -0
  66. helm/benchmark/metrics/image_generation/fractal_dimension/test_fractal_dimension_util.py +33 -0
  67. helm/benchmark/metrics/image_generation/fractal_dimension_metric.py +50 -0
  68. helm/benchmark/metrics/image_generation/gender_metrics.py +58 -0
  69. helm/benchmark/metrics/image_generation/image_critique_metrics.py +284 -0
  70. helm/benchmark/metrics/image_generation/lpips_metrics.py +82 -0
  71. helm/benchmark/metrics/image_generation/multi_scale_ssim_metrics.py +82 -0
  72. helm/benchmark/metrics/image_generation/nsfw_detector.py +96 -0
  73. helm/benchmark/metrics/image_generation/nsfw_metrics.py +103 -0
  74. helm/benchmark/metrics/image_generation/nudity_metrics.py +38 -0
  75. helm/benchmark/metrics/image_generation/photorealism_critique_metrics.py +153 -0
  76. helm/benchmark/metrics/image_generation/psnr_metrics.py +78 -0
  77. helm/benchmark/metrics/image_generation/q16/__init__.py +0 -0
  78. helm/benchmark/metrics/image_generation/q16/q16_toxicity_detector.py +90 -0
  79. helm/benchmark/metrics/image_generation/q16/test_q16.py +18 -0
  80. helm/benchmark/metrics/image_generation/q16_toxicity_metrics.py +48 -0
  81. helm/benchmark/metrics/image_generation/skin_tone_metrics.py +164 -0
  82. helm/benchmark/metrics/image_generation/uiqi_metrics.py +92 -0
  83. helm/benchmark/metrics/image_generation/watermark/__init__.py +0 -0
  84. helm/benchmark/metrics/image_generation/watermark/test_watermark_detector.py +16 -0
  85. helm/benchmark/metrics/image_generation/watermark/watermark_detector.py +87 -0
  86. helm/benchmark/metrics/image_generation/watermark_metrics.py +48 -0
  87. helm/benchmark/metrics/instruction_following_critique_metrics.py +3 -1
  88. helm/benchmark/metrics/language_modeling_metrics.py +99 -0
  89. helm/benchmark/metrics/machine_translation_metrics.py +89 -0
  90. helm/benchmark/metrics/metric.py +93 -172
  91. helm/benchmark/metrics/metric_name.py +0 -1
  92. helm/benchmark/metrics/metric_service.py +16 -0
  93. helm/benchmark/metrics/paraphrase_generation_metrics.py +3 -4
  94. helm/benchmark/metrics/ranking_metrics.py +2 -2
  95. helm/benchmark/metrics/reference_metric.py +148 -0
  96. helm/benchmark/metrics/summac/model_summac.py +0 -2
  97. helm/benchmark/metrics/summarization_metrics.py +2 -2
  98. helm/benchmark/metrics/test_classification_metrics.py +8 -5
  99. helm/benchmark/metrics/test_disinformation_metrics.py +78 -0
  100. helm/benchmark/metrics/{test_basic_metrics.py → test_evaluate_reference_metrics.py} +5 -1
  101. helm/benchmark/metrics/test_metric.py +2 -2
  102. helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py +10 -2
  103. helm/benchmark/metrics/toxicity_metrics.py +1 -1
  104. helm/benchmark/metrics/toxicity_utils.py +23 -0
  105. helm/benchmark/metrics/unitxt_metrics.py +81 -0
  106. helm/benchmark/metrics/vision_language/__init__.py +0 -0
  107. helm/benchmark/metrics/vision_language/emd_utils.py +341 -0
  108. helm/benchmark/metrics/vision_language/image_metrics.py +450 -0
  109. helm/benchmark/metrics/vision_language/image_utils.py +100 -0
  110. helm/benchmark/model_deployment_registry.py +74 -0
  111. helm/benchmark/model_metadata_registry.py +36 -0
  112. helm/benchmark/multi_gpu_runner.py +133 -0
  113. helm/benchmark/presentation/create_plots.py +8 -7
  114. helm/benchmark/presentation/run_display.py +26 -10
  115. helm/benchmark/presentation/schema.py +15 -40
  116. helm/benchmark/presentation/summarize.py +119 -79
  117. helm/benchmark/presentation/table.py +8 -8
  118. helm/benchmark/presentation/test_contamination.py +2 -2
  119. helm/benchmark/presentation/test_run_entry.py +1 -2
  120. helm/benchmark/presentation/test_summarize.py +3 -3
  121. helm/benchmark/run.py +54 -26
  122. helm/benchmark/run_expander.py +214 -16
  123. helm/benchmark/run_spec.py +93 -0
  124. helm/benchmark/run_spec_factory.py +162 -0
  125. helm/benchmark/run_specs/__init__.py +0 -0
  126. helm/benchmark/run_specs/classic_run_specs.py +1510 -0
  127. helm/benchmark/run_specs/cleva_run_specs.py +277 -0
  128. helm/benchmark/run_specs/decodingtrust_run_specs.py +314 -0
  129. helm/benchmark/run_specs/heim_run_specs.py +623 -0
  130. helm/benchmark/run_specs/instruction_following_run_specs.py +129 -0
  131. helm/benchmark/run_specs/lite_run_specs.py +307 -0
  132. helm/benchmark/run_specs/simple_run_specs.py +104 -0
  133. helm/benchmark/run_specs/unitxt_run_specs.py +42 -0
  134. helm/benchmark/run_specs/vlm_run_specs.py +501 -0
  135. helm/benchmark/runner.py +51 -57
  136. helm/benchmark/runner_config_registry.py +21 -0
  137. helm/benchmark/scenarios/bbq_scenario.py +1 -1
  138. helm/benchmark/scenarios/bold_scenario.py +2 -2
  139. helm/benchmark/scenarios/code_scenario.py +1 -0
  140. helm/benchmark/scenarios/decodingtrust_adv_demonstration_scenario.py +169 -0
  141. helm/benchmark/scenarios/decodingtrust_adv_robustness_scenario.py +121 -0
  142. helm/benchmark/scenarios/decodingtrust_fairness_scenario.py +77 -0
  143. helm/benchmark/scenarios/decodingtrust_machine_ethics_scenario.py +324 -0
  144. helm/benchmark/scenarios/decodingtrust_ood_robustness_scenario.py +204 -0
  145. helm/benchmark/scenarios/decodingtrust_privacy_scenario.py +559 -0
  146. helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py +67 -0
  147. helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py +78 -0
  148. helm/benchmark/scenarios/dialogue_scenarios.py +0 -1
  149. helm/benchmark/scenarios/image_generation/__init__.py +0 -0
  150. helm/benchmark/scenarios/image_generation/common_syntactic_processes_scenario.py +105 -0
  151. helm/benchmark/scenarios/image_generation/cub200_scenario.py +95 -0
  152. helm/benchmark/scenarios/image_generation/daily_dalle_scenario.py +124 -0
  153. helm/benchmark/scenarios/image_generation/demographic_stereotypes_scenario.py +82 -0
  154. helm/benchmark/scenarios/image_generation/detection_scenario.py +83 -0
  155. helm/benchmark/scenarios/image_generation/draw_bench_scenario.py +74 -0
  156. helm/benchmark/scenarios/image_generation/i2p_scenario.py +57 -0
  157. helm/benchmark/scenarios/image_generation/landing_page_scenario.py +46 -0
  158. helm/benchmark/scenarios/image_generation/logos_scenario.py +223 -0
  159. helm/benchmark/scenarios/image_generation/magazine_cover_scenario.py +91 -0
  160. helm/benchmark/scenarios/image_generation/mental_disorders_scenario.py +46 -0
  161. helm/benchmark/scenarios/image_generation/mscoco_scenario.py +91 -0
  162. helm/benchmark/scenarios/image_generation/paint_skills_scenario.py +72 -0
  163. helm/benchmark/scenarios/image_generation/parti_prompts_scenario.py +94 -0
  164. helm/benchmark/scenarios/image_generation/radiology_scenario.py +42 -0
  165. helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py +52 -0
  166. helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py +124 -0
  167. helm/benchmark/scenarios/image_generation/winoground_scenario.py +62 -0
  168. helm/benchmark/scenarios/imdb_scenario.py +0 -1
  169. helm/benchmark/scenarios/live_qa_scenario.py +94 -0
  170. helm/benchmark/scenarios/lm_entry_scenario.py +185 -0
  171. helm/benchmark/scenarios/math_scenario.py +19 -2
  172. helm/benchmark/scenarios/medication_qa_scenario.py +60 -0
  173. helm/benchmark/scenarios/numeracy_scenario.py +1 -1
  174. helm/benchmark/scenarios/opinions_qa_scenario.py +0 -4
  175. helm/benchmark/scenarios/scenario.py +4 -0
  176. helm/benchmark/scenarios/simple_scenarios.py +122 -1
  177. helm/benchmark/scenarios/test_math_scenario.py +6 -0
  178. helm/benchmark/scenarios/test_scenario.py +6 -3
  179. helm/benchmark/scenarios/test_simple_scenarios.py +50 -0
  180. helm/benchmark/scenarios/thai_exam_scenario.py +135 -0
  181. helm/benchmark/scenarios/unitxt_scenario.py +56 -0
  182. helm/benchmark/scenarios/verifiability_judgment_scenario.py +3 -1
  183. helm/benchmark/scenarios/vicuna_scenario.py +1 -1
  184. helm/benchmark/scenarios/vision_language/bingo_scenario.py +103 -0
  185. helm/benchmark/scenarios/vision_language/hateful_memes_scenario.py +92 -0
  186. helm/benchmark/scenarios/vision_language/heim_human_eval_scenario.py +113 -0
  187. helm/benchmark/scenarios/vision_language/image2structure/__init__.py +0 -0
  188. helm/benchmark/scenarios/vision_language/image2structure/chart2csv_scenario.py +55 -0
  189. helm/benchmark/scenarios/vision_language/image2structure/image2structure_scenario.py +214 -0
  190. helm/benchmark/scenarios/vision_language/image2structure/latex_scenario.py +25 -0
  191. helm/benchmark/scenarios/vision_language/image2structure/musicsheet_scenario.py +20 -0
  192. helm/benchmark/scenarios/vision_language/image2structure/utils_latex.py +347 -0
  193. helm/benchmark/scenarios/vision_language/image2structure/webpage/__init__.py +0 -0
  194. helm/benchmark/scenarios/vision_language/image2structure/webpage/driver.py +84 -0
  195. helm/benchmark/scenarios/vision_language/image2structure/webpage/jekyll_server.py +182 -0
  196. helm/benchmark/scenarios/vision_language/image2structure/webpage/utils.py +31 -0
  197. helm/benchmark/scenarios/vision_language/image2structure/webpage_scenario.py +225 -0
  198. helm/benchmark/scenarios/vision_language/mementos_scenario.py +124 -0
  199. helm/benchmark/scenarios/vision_language/mme_scenario.py +145 -0
  200. helm/benchmark/scenarios/vision_language/mmmu_scenario.py +187 -0
  201. helm/benchmark/scenarios/vision_language/multipanelvqa_scenario.py +169 -0
  202. helm/benchmark/scenarios/vision_language/pope_scenario.py +104 -0
  203. helm/benchmark/scenarios/vision_language/seed_bench_scenario.py +129 -0
  204. helm/benchmark/scenarios/vision_language/unicorn_scenario.py +108 -0
  205. helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py +1 -2
  206. helm/benchmark/scenarios/vision_language/vqa_scenario.py +1 -1
  207. helm/benchmark/scenarios/wmt_14_scenario.py +1 -1
  208. helm/benchmark/server.py +24 -1
  209. helm/benchmark/slurm_runner.py +70 -49
  210. helm/benchmark/static/benchmarking.js +1 -1
  211. helm/benchmark/static/schema_classic.yaml +258 -1066
  212. helm/benchmark/static/schema_instruction_following.yaml +210 -0
  213. helm/benchmark/static/schema_lite.yaml +2 -227
  214. helm/benchmark/static/schema_mmlu.yaml +1507 -0
  215. helm/benchmark/static/schema_unitxt.yaml +428 -0
  216. helm/benchmark/static/schema_vlm.yaml +576 -0
  217. helm/benchmark/static_build/assets/01-694cb9b7.png +0 -0
  218. helm/benchmark/static_build/assets/ai21-0eb91ec3.png +0 -0
  219. helm/benchmark/static_build/assets/aleph-alpha-7ce10034.png +0 -0
  220. helm/benchmark/static_build/assets/anthropic-70d8bc39.png +0 -0
  221. helm/benchmark/static_build/assets/bigscience-7f0400c0.png +0 -0
  222. helm/benchmark/static_build/assets/cohere-3550c6cb.png +0 -0
  223. helm/benchmark/static_build/assets/crfm-logo-74391ab8.png +0 -0
  224. helm/benchmark/static_build/assets/eleutherai-b9451114.png +0 -0
  225. helm/benchmark/static_build/assets/google-06d997ad.png +0 -0
  226. helm/benchmark/static_build/assets/heim-logo-3e5e3aa4.png +0 -0
  227. helm/benchmark/static_build/assets/helm-logo-simple-2ed5400b.png +0 -0
  228. helm/benchmark/static_build/assets/helmhero-28e90f4d.png +0 -0
  229. helm/benchmark/static_build/assets/index-5088afcb.css +1 -0
  230. helm/benchmark/static_build/assets/index-d839df55.js +9 -0
  231. helm/benchmark/static_build/assets/meta-5580e9f1.png +0 -0
  232. helm/benchmark/static_build/assets/microsoft-f5ee5016.png +0 -0
  233. helm/benchmark/static_build/assets/mistral-18e1be23.png +0 -0
  234. helm/benchmark/static_build/assets/nvidia-86fa75c1.png +0 -0
  235. helm/benchmark/static_build/assets/openai-3f8653e4.png +0 -0
  236. helm/benchmark/static_build/assets/react-d4a0b69b.js +85 -0
  237. helm/benchmark/static_build/assets/recharts-6d337683.js +97 -0
  238. helm/benchmark/static_build/assets/tii-24de195c.png +0 -0
  239. helm/benchmark/static_build/assets/together-a665a35b.png +0 -0
  240. helm/benchmark/static_build/assets/tremor-54a99cc4.js +10 -0
  241. helm/benchmark/static_build/assets/tsinghua-keg-97d4b395.png +0 -0
  242. helm/benchmark/static_build/assets/vhelm-framework-cde7618a.png +0 -0
  243. helm/benchmark/static_build/assets/vhelm-model-6d812526.png +0 -0
  244. helm/benchmark/static_build/assets/yandex-38e09d70.png +0 -0
  245. helm/benchmark/static_build/config.js +4 -0
  246. helm/benchmark/static_build/index.html +20 -0
  247. helm/benchmark/test_data_preprocessor.py +3 -3
  248. helm/benchmark/test_model_deployment_definition.py +14 -16
  249. helm/benchmark/test_run_expander.py +1 -1
  250. helm/benchmark/window_services/ai21_window_service.py +22 -33
  251. helm/benchmark/window_services/cohere_window_service.py +1 -63
  252. helm/benchmark/window_services/default_window_service.py +2 -44
  253. helm/benchmark/window_services/encoder_decoder_window_service.py +0 -11
  254. helm/benchmark/window_services/ice_window_service.py +0 -34
  255. helm/benchmark/window_services/image_generation/__init__.py +0 -0
  256. helm/benchmark/window_services/image_generation/clip_window_service.py +15 -0
  257. helm/benchmark/window_services/image_generation/lexica_search_window_service.py +9 -0
  258. helm/benchmark/window_services/image_generation/openai_dalle_window_service.py +9 -0
  259. helm/benchmark/window_services/image_generation/test_clip_window_service.py +29 -0
  260. helm/benchmark/window_services/image_generation/test_openai_dalle_window_service.py +30 -0
  261. helm/benchmark/window_services/local_window_service.py +21 -4
  262. helm/benchmark/window_services/test_anthropic_window_service.py +2 -1
  263. helm/benchmark/window_services/test_bloom_window_service.py +2 -1
  264. helm/benchmark/window_services/test_cohere_window_service.py +2 -1
  265. helm/benchmark/window_services/test_flan_t5_window_service.py +2 -1
  266. helm/benchmark/window_services/test_gpt2_window_service.py +2 -2
  267. helm/benchmark/window_services/test_gpt4_window_service.py +2 -1
  268. helm/benchmark/window_services/test_gptj_window_service.py +3 -2
  269. helm/benchmark/window_services/test_gptneox_window_service.py +3 -2
  270. helm/benchmark/window_services/test_ice_window_service.py +2 -1
  271. helm/benchmark/window_services/test_openai_window_service.py +2 -1
  272. helm/benchmark/window_services/test_opt_window_service.py +3 -2
  273. helm/benchmark/window_services/test_palmyra_window_service.py +2 -1
  274. helm/benchmark/window_services/test_t0pp_window_service.py +2 -1
  275. helm/benchmark/window_services/test_t511b_window_service.py +2 -1
  276. helm/benchmark/window_services/test_ul2_window_service.py +2 -1
  277. helm/benchmark/window_services/test_utils.py +3 -2
  278. helm/benchmark/window_services/test_yalm_window_service.py +2 -1
  279. helm/benchmark/window_services/window_service.py +42 -0
  280. helm/benchmark/window_services/window_service_factory.py +4 -1
  281. helm/benchmark/window_services/yalm_window_service.py +0 -27
  282. helm/clients/__init__.py +0 -0
  283. helm/{proxy/clients → clients}/ai21_client.py +3 -9
  284. helm/clients/aleph_alpha_client.py +112 -0
  285. helm/{proxy/clients → clients}/anthropic_client.py +203 -18
  286. helm/{proxy/clients → clients}/auto_client.py +59 -31
  287. helm/clients/bedrock_client.py +128 -0
  288. helm/clients/bedrock_utils.py +72 -0
  289. helm/{proxy/clients → clients}/client.py +65 -7
  290. helm/clients/clip_score_client.py +49 -0
  291. helm/clients/clip_scorers/__init__.py +0 -0
  292. helm/clients/clip_scorers/base_clip_scorer.py +18 -0
  293. helm/clients/clip_scorers/clip_scorer.py +50 -0
  294. helm/clients/clip_scorers/multilingual_clip_scorer.py +50 -0
  295. helm/{proxy/clients → clients}/cohere_client.py +4 -11
  296. helm/clients/gcs_client.py +82 -0
  297. helm/{proxy/clients → clients}/google_client.py +5 -5
  298. helm/clients/google_translate_client.py +35 -0
  299. helm/{proxy/clients → clients}/http_model_client.py +5 -7
  300. helm/{proxy/clients → clients}/huggingface_client.py +43 -64
  301. helm/clients/image_generation/__init__.py +0 -0
  302. helm/clients/image_generation/adobe_vision_client.py +78 -0
  303. helm/clients/image_generation/aleph_alpha_image_generation_client.py +98 -0
  304. helm/clients/image_generation/cogview2/__init__.py +0 -0
  305. helm/clients/image_generation/cogview2/coglm_strategy.py +96 -0
  306. helm/clients/image_generation/cogview2/coglm_utils.py +82 -0
  307. helm/clients/image_generation/cogview2/sr_pipeline/__init__.py +15 -0
  308. helm/clients/image_generation/cogview2/sr_pipeline/direct_sr.py +96 -0
  309. helm/clients/image_generation/cogview2/sr_pipeline/dsr_model.py +254 -0
  310. helm/clients/image_generation/cogview2/sr_pipeline/dsr_sampling.py +190 -0
  311. helm/clients/image_generation/cogview2/sr_pipeline/iterative_sr.py +141 -0
  312. helm/clients/image_generation/cogview2/sr_pipeline/itersr_model.py +269 -0
  313. helm/clients/image_generation/cogview2/sr_pipeline/itersr_sampling.py +120 -0
  314. helm/clients/image_generation/cogview2/sr_pipeline/sr_group.py +42 -0
  315. helm/clients/image_generation/cogview2_client.py +191 -0
  316. helm/clients/image_generation/dalle2_client.py +192 -0
  317. helm/clients/image_generation/dalle3_client.py +108 -0
  318. helm/clients/image_generation/dalle_mini/__init__.py +3 -0
  319. helm/clients/image_generation/dalle_mini/data.py +442 -0
  320. helm/clients/image_generation/dalle_mini/model/__init__.py +5 -0
  321. helm/clients/image_generation/dalle_mini/model/configuration.py +175 -0
  322. helm/clients/image_generation/dalle_mini/model/modeling.py +1834 -0
  323. helm/clients/image_generation/dalle_mini/model/partitions.py +84 -0
  324. helm/clients/image_generation/dalle_mini/model/processor.py +63 -0
  325. helm/clients/image_generation/dalle_mini/model/text.py +251 -0
  326. helm/clients/image_generation/dalle_mini/model/tokenizer.py +9 -0
  327. helm/clients/image_generation/dalle_mini/model/utils.py +29 -0
  328. helm/clients/image_generation/dalle_mini/vqgan_jax/__init__.py +1 -0
  329. helm/clients/image_generation/dalle_mini/vqgan_jax/configuration_vqgan.py +40 -0
  330. helm/clients/image_generation/dalle_mini/vqgan_jax/convert_pt_model_to_jax.py +107 -0
  331. helm/clients/image_generation/dalle_mini/vqgan_jax/modeling_flax_vqgan.py +610 -0
  332. helm/clients/image_generation/dalle_mini_client.py +190 -0
  333. helm/clients/image_generation/deep_floyd_client.py +78 -0
  334. helm/clients/image_generation/huggingface_diffusers_client.py +249 -0
  335. helm/clients/image_generation/image_generation_client_utils.py +9 -0
  336. helm/clients/image_generation/lexica_client.py +86 -0
  337. helm/clients/image_generation/mindalle/__init__.py +0 -0
  338. helm/clients/image_generation/mindalle/models/__init__.py +216 -0
  339. helm/clients/image_generation/mindalle/models/stage1/__init__.py +0 -0
  340. helm/clients/image_generation/mindalle/models/stage1/layers.py +312 -0
  341. helm/clients/image_generation/mindalle/models/stage1/vqgan.py +103 -0
  342. helm/clients/image_generation/mindalle/models/stage2/__init__.py +0 -0
  343. helm/clients/image_generation/mindalle/models/stage2/layers.py +144 -0
  344. helm/clients/image_generation/mindalle/models/stage2/transformer.py +268 -0
  345. helm/clients/image_generation/mindalle/models/tokenizer.py +30 -0
  346. helm/clients/image_generation/mindalle/utils/__init__.py +3 -0
  347. helm/clients/image_generation/mindalle/utils/config.py +129 -0
  348. helm/clients/image_generation/mindalle/utils/sampling.py +149 -0
  349. helm/clients/image_generation/mindalle/utils/utils.py +89 -0
  350. helm/clients/image_generation/mindalle_client.py +115 -0
  351. helm/clients/image_generation/nudity_check_client.py +64 -0
  352. helm/clients/image_generation/together_image_generation_client.py +111 -0
  353. helm/{proxy/clients → clients}/lit_gpt_client.py +4 -4
  354. helm/{proxy/clients → clients}/megatron_client.py +5 -5
  355. helm/clients/mistral_client.py +134 -0
  356. helm/clients/moderation_api_client.py +109 -0
  357. helm/clients/open_lm_client.py +43 -0
  358. helm/clients/openai_client.py +302 -0
  359. helm/{proxy/clients → clients}/palmyra_client.py +6 -8
  360. helm/{proxy/clients → clients}/perspective_api_client.py +7 -8
  361. helm/clients/simple_client.py +64 -0
  362. helm/{proxy/clients → clients}/test_auto_client.py +13 -15
  363. helm/clients/test_client.py +100 -0
  364. helm/{proxy/clients → clients}/test_huggingface_client.py +15 -16
  365. helm/clients/test_simple_client.py +19 -0
  366. helm/{proxy/clients → clients}/test_together_client.py +20 -8
  367. helm/{proxy/clients → clients}/together_client.py +12 -72
  368. helm/clients/vertexai_client.py +391 -0
  369. helm/clients/vision_language/__init__.py +0 -0
  370. helm/clients/vision_language/huggingface_vlm_client.py +104 -0
  371. helm/{proxy/clients → clients}/vision_language/idefics_client.py +53 -48
  372. helm/clients/vision_language/open_flamingo/__init__.py +2 -0
  373. helm/clients/vision_language/open_flamingo/src/__init__.py +0 -0
  374. helm/clients/vision_language/open_flamingo/src/factory.py +147 -0
  375. helm/clients/vision_language/open_flamingo/src/flamingo.py +337 -0
  376. helm/clients/vision_language/open_flamingo/src/flamingo_lm.py +155 -0
  377. helm/clients/vision_language/open_flamingo/src/helpers.py +267 -0
  378. helm/clients/vision_language/open_flamingo/src/utils.py +47 -0
  379. helm/clients/vision_language/open_flamingo_client.py +155 -0
  380. helm/clients/vision_language/qwen_vlm_client.py +171 -0
  381. helm/clients/vllm_client.py +46 -0
  382. helm/common/cache.py +16 -4
  383. helm/common/cache_backend_config.py +47 -0
  384. helm/common/clip_score_request.py +41 -0
  385. helm/common/file_caches/__init__.py +0 -0
  386. helm/common/file_caches/file_cache.py +16 -0
  387. helm/common/file_caches/local_file_cache.py +61 -0
  388. helm/common/file_caches/test_local_file_cache.py +25 -0
  389. helm/common/file_upload_request.py +27 -0
  390. helm/common/general.py +1 -1
  391. helm/common/image_generation_parameters.py +25 -0
  392. helm/common/images_utils.py +24 -1
  393. helm/common/key_value_store.py +35 -4
  394. helm/common/media_object.py +13 -0
  395. helm/common/moderations_api_request.py +71 -0
  396. helm/common/mongo_key_value_store.py +3 -3
  397. helm/common/multimodal_request_utils.py +31 -0
  398. helm/common/nudity_check_request.py +29 -0
  399. helm/common/request.py +15 -17
  400. helm/common/test_general.py +6 -0
  401. helm/common/tokenization_request.py +1 -1
  402. helm/config/model_deployments.yaml +1069 -546
  403. helm/config/model_metadata.yaml +753 -31
  404. helm/config/tokenizer_configs.yaml +142 -43
  405. helm/proxy/accounts.py +31 -4
  406. helm/proxy/critique/mechanical_turk_critique_importer.py +3 -0
  407. helm/proxy/critique/model_critique_client.py +8 -6
  408. helm/proxy/example_queries.py +29 -17
  409. helm/proxy/server.py +70 -5
  410. helm/proxy/services/remote_service.py +31 -0
  411. helm/proxy/services/server_service.py +96 -16
  412. helm/proxy/services/service.py +30 -0
  413. helm/proxy/services/test_remote_service.py +4 -3
  414. helm/proxy/services/test_service.py +0 -12
  415. helm/proxy/test_accounts.py +32 -0
  416. helm/proxy/token_counters/auto_token_counter.py +37 -37
  417. helm/proxy/token_counters/test_auto_token_counter.py +164 -0
  418. helm/proxy/token_counters/token_counter.py +3 -5
  419. helm/tokenizers/__init__.py +0 -0
  420. helm/{proxy/tokenizers → tokenizers}/ai21_tokenizer.py +3 -3
  421. helm/{proxy/tokenizers → tokenizers}/anthropic_tokenizer.py +1 -1
  422. helm/{proxy/tokenizers → tokenizers}/auto_tokenizer.py +6 -9
  423. helm/{proxy/tokenizers → tokenizers}/cohere_tokenizer.py +1 -1
  424. helm/{proxy/tokenizers → tokenizers}/http_model_tokenizer.py +3 -3
  425. helm/{proxy/tokenizers → tokenizers}/huggingface_tokenizer.py +7 -26
  426. helm/tokenizers/simple_tokenizer.py +33 -0
  427. helm/{proxy/tokenizers → tokenizers}/test_anthropic_tokenizer.py +1 -1
  428. helm/{proxy/tokenizers → tokenizers}/test_huggingface_tokenizer.py +3 -0
  429. helm/tokenizers/test_simple_tokenizer.py +33 -0
  430. helm/{proxy/tokenizers → tokenizers}/vertexai_tokenizer.py +1 -1
  431. helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer.py +5 -3
  432. helm/tokenizers/yalm_tokenizer_data/__init__.py +0 -0
  433. helm/tokenizers/yalm_tokenizer_data/voc_100b.sp +0 -0
  434. helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/yalm_tokenizer.py +1 -1
  435. crfm_helm-0.4.0.dist-info/RECORD +0 -397
  436. helm/benchmark/run_specs.py +0 -2762
  437. helm/benchmark/test_model_properties.py +0 -1570
  438. helm/benchmark/vlm_run_specs.py +0 -97
  439. helm/benchmark/window_services/flan_t5_window_service.py +0 -29
  440. helm/benchmark/window_services/gpt2_window_service.py +0 -32
  441. helm/benchmark/window_services/huggingface_window_service.py +0 -60
  442. helm/benchmark/window_services/t0pp_window_service.py +0 -35
  443. helm/benchmark/window_services/t511b_window_service.py +0 -30
  444. helm/benchmark/window_services/test_mt_nlg_window_service.py +0 -48
  445. helm/benchmark/window_services/ul2_window_service.py +0 -30
  446. helm/benchmark/window_services/wider_ai21_window_service.py +0 -24
  447. helm/common/cache_utils.py +0 -14
  448. helm/proxy/clients/aleph_alpha_client.py +0 -95
  449. helm/proxy/clients/goose_ai_client.py +0 -99
  450. helm/proxy/clients/microsoft_client.py +0 -180
  451. helm/proxy/clients/openai_client.py +0 -206
  452. helm/proxy/clients/simple_client.py +0 -60
  453. helm/proxy/clients/test_client.py +0 -49
  454. helm/proxy/clients/vertexai_client.py +0 -115
  455. helm/proxy/token_counters/ai21_token_counter.py +0 -20
  456. helm/proxy/token_counters/cohere_token_counter.py +0 -13
  457. helm/proxy/token_counters/free_token_counter.py +0 -12
  458. helm/proxy/token_counters/gooseai_token_counter.py +0 -24
  459. helm/proxy/token_counters/openai_token_counter.py +0 -22
  460. helm/proxy/token_counters/test_ai21_token_counter.py +0 -88
  461. helm/proxy/token_counters/test_openai_token_counter.py +0 -81
  462. helm/proxy/tokenizers/simple_tokenizer.py +0 -32
  463. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.0.dist-info}/LICENSE +0 -0
  464. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.0.dist-info}/entry_points.txt +0 -0
  465. {crfm_helm-0.4.0.dist-info → crfm_helm-0.5.0.dist-info}/top_level.txt +0 -0
  466. /helm/{proxy/clients → benchmark/annotation}/__init__.py +0 -0
  467. /helm/{proxy/clients/vision_language → benchmark/annotation/image2structure}/__init__.py +0 -0
  468. /helm/{proxy/tokenizers → benchmark/metrics/image_generation}/__init__.py +0 -0
  469. /helm/{proxy/tokenizers/yalm_tokenizer_data → benchmark/metrics/image_generation/detectors}/__init__.py +0 -0
  470. /helm/{proxy/clients → clients}/ai21_utils.py +0 -0
  471. /helm/{proxy/clients → clients}/cohere_utils.py +0 -0
  472. /helm/{proxy/clients → clients}/lit_gpt_generate.py +0 -0
  473. /helm/{proxy/clients → clients}/toxicity_classifier_client.py +0 -0
  474. /helm/{proxy/tokenizers → tokenizers}/aleph_alpha_tokenizer.py +0 -0
  475. /helm/{proxy/tokenizers → tokenizers}/caching_tokenizer.py +0 -0
  476. /helm/{proxy/tokenizers → tokenizers}/ice_tokenizer.py +0 -0
  477. /helm/{proxy/tokenizers → tokenizers}/lit_gpt_tokenizer.py +0 -0
  478. /helm/{proxy/tokenizers → tokenizers}/test_ice_tokenizer.py +0 -0
  479. /helm/{proxy/tokenizers → tokenizers}/test_yalm_tokenizer.py +0 -0
  480. /helm/{proxy/tokenizers → tokenizers}/tiktoken_tokenizer.py +0 -0
  481. /helm/{proxy/tokenizers → tokenizers}/tokenizer.py +0 -0
  482. /helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/test_yalm_tokenizer.py +0 -0
crfm_helm-0.4.0.dist-info/RECORD DELETED
@@ -1,397 +0,0 @@
1
- helm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- helm/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
- helm/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
- helm/benchmark/config_registry.py,sha256=jEUik24cpHer0KrKTQQGcRf6p4NW0peRwaXToHjqnZE,1295
5
- helm/benchmark/data_preprocessor.py,sha256=aNdM-o2t4qkLIQHiQeWUFg03DjjJ8HTBIphYCK8pXVo,2173
6
- helm/benchmark/executor.py,sha256=zi3YzO1rFHegwWYZcu7HC6ClqgqjhzSAdVgIOMk20G8,3586
7
- helm/benchmark/huggingface_registration.py,sha256=kDnnXTKAE6tPEGZRaZ-Dkly32ooGNB03Wbtm1A1uX8o,3350
8
- helm/benchmark/model_deployment_registry.py,sha256=Hftahy-eHm6pI815JEsdNVlv-ln4z2ISl4ojo3fAIYw,5460
9
- helm/benchmark/model_metadata_registry.py,sha256=1NaHyk1ojwSYPf8UH-mAqEZZSoiqlKUKn8MtQK1robI,6458
10
- helm/benchmark/run.py,sha256=qGdBjTTlYN996CAs5WSqf2yFFLNN85TP53FR0tjhnSw,11409
11
- helm/benchmark/run_expander.py,sha256=6QhHcPYDzOnV4gBF1QPw7vHlSnZjv6yZjT_0SzEqoMk,41365
12
- helm/benchmark/run_specs.py,sha256=RC3BN-fPGHKgIu03Ew07p7EYhzxWsr4beU5Xz-p82Xg,104887
13
- helm/benchmark/runner.py,sha256=S38RHCOObuoUyfF7OAB0H_mgSaTz-Ptzwlz2oj4EmxE,14262
14
- helm/benchmark/server.py,sha256=iBODLZC_-GVFNUvkPBHKFEhtSdZCrgDwvs5ka2NQcmg,5127
15
- helm/benchmark/slurm_jobs.py,sha256=eNCAoaWDfT0Wk32ZJRIGo-x8kgjhDPnPB4Xrvw_eLB0,3225
16
- helm/benchmark/slurm_runner.py,sha256=_5tqyap8EbqOXKtT_8ZuB3ChVSlsNTNSSaO-axSNkYM,15795
17
- helm/benchmark/test_data_preprocessor.py,sha256=xIwFuhb7WpdGLfFq9gfBEzPlsmyIEZMuHW4iPtq2HEY,2180
18
- helm/benchmark/test_model_deployment_definition.py,sha256=EZ9WqI9y6u-XfK7XRsk0Ew_o5tnoyvOL0UMNkKDt8NM,4537
19
- helm/benchmark/test_model_properties.py,sha256=ioi7CotXyXtlhP6XOEVQYQqcMvnh5gX2QlcdjbSuY_g,68560
20
- helm/benchmark/test_run_expander.py,sha256=nVxlsu5mFC-omMTTSDR0CHAaDDnI1sXbLxjCj4Xuu6I,1161
21
- helm/benchmark/tokenizer_config_registry.py,sha256=ZOImg38ta0FXZYAWna6q7A5xrG2mU7Ofr-8j4EqGlUY,1585
22
- helm/benchmark/vlm_run_specs.py,sha256=aC5p74RzpJenyVb8f9JcIyUnYa2rl5yQuFDbrLggBog,3100
23
- helm/benchmark/adaptation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
- helm/benchmark/adaptation/adapter_spec.py,sha256=9nyHrSrC0EoS6c1q8ocY4cnvdiCtU4Ndl-GONr1fe4g,3295
25
- helm/benchmark/adaptation/prompt.py,sha256=n0Ka3RGSWMr3CBnJrPNPy626x9TJE3k677wKbG8hO9A,2133
26
- helm/benchmark/adaptation/request_state.py,sha256=o3OpZbB0TJFiZ2Nmhvg3vWmByaUSYTffT_WnoNb7w68,2712
27
- helm/benchmark/adaptation/scenario_state.py,sha256=ZflBuNgvN0JqUhshFcy0kTweO1WJs6j5UCaTxWTMe0o,1747
28
- helm/benchmark/adaptation/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
- helm/benchmark/adaptation/adapters/adapter.py,sha256=rFqfjTsha3r6CgRat2FKwIJb3pl-IyyoGUznwQtcHag,1236
30
- helm/benchmark/adaptation/adapters/adapter_factory.py,sha256=VUBOfOWKC8k0FimJ5ZW8CZZVK0W33eg801wK1ZhYIjg,2627
31
- helm/benchmark/adaptation/adapters/binary_ranking_adapter.py,sha256=zF5KcGip5CAMfg5mzisYQ2jlIuChbraujN-UaxCV8E8,5782
32
- helm/benchmark/adaptation/adapters/generation_adapter.py,sha256=IqCFpU3r4YuIeqbTgK0MPJtvLNXomkIpgld5Jj7ciZw,2059
33
- helm/benchmark/adaptation/adapters/in_context_learning_adapter.py,sha256=wJO8-vOMk07tH3X6PCo4LXC6eSZpSzvMxdKObq6-4Cg,14244
34
- helm/benchmark/adaptation/adapters/language_modeling_adapter.py,sha256=Vs0AU-laV89aPF9CZaOYG8nf8Ukpmc5ozXNZ-Amu5b0,14969
35
- helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py,sha256=VJ66MfIGQWJg0VXCV0MJEMwF9Jx1DeJ7RxsgYlOTx_4,1889
36
- helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py,sha256=k8wSxv9pK8wtbQNBzWYPkGEUKJb8tcVi41Y1M3fPQT4,3985
37
- helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py,sha256=aMe-y4iiyEnM7_bqIoddeZBsVLoDxMmjKY2eZKB6Y2Q,2156
38
- helm/benchmark/adaptation/adapters/test_adapter.py,sha256=u-c7K6VH6yvymRtPjoWRBIjk0vs41g7IWOfv1w9Sxn0,620
39
- helm/benchmark/adaptation/adapters/test_generation_adapter.py,sha256=rbZlC_xHEKCXo5GnDt7C8856fVDjX18QGpMvvn0RRaE,11765
40
- helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py,sha256=VCD-9EKDi9_xElUKU6cjAgUoPTqx5UGF2tiXAxsV45E,8490
41
- helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py,sha256=CiIpclOvpIDigOVS8NgjCQOqt1olcxs5xPFVMlmuckM,7379
42
- helm/benchmark/adaptation/adapters/multimodal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
- helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py,sha256=o7CGClyVWYOuJ4G56-whq5fTvCr7QIn51Mo6DTdvwg0,1881
44
- helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py,sha256=9_9PgNIiJDoTm1NIhCKWytCYe3hHzHJUhaAdd8hM0PQ,6300
45
- helm/benchmark/adaptation/adapters/multimodal/multimodal_prompt.py,sha256=5jdVVfWf0lBErwzDamIHozZSbf1mEkpmACX8mPlHJc0,1907
46
- helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py,sha256=GANySWgsdt5SyajawgMPPqCF82bsd27VBHDfGngupc0,10047
47
- helm/benchmark/adaptation/adapters/multimodal/test_multimodal_prompt.py,sha256=2OZhhOtEZ7RvItHZD4AGsefXvVUwEQZL2T-ESHPsvdI,3482
48
- helm/benchmark/augmentations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
49
- helm/benchmark/augmentations/cleva_perturbation.py,sha256=arUkY_luc274YEMZocOos9rpAZVbEFZphbMlobAxTy0,29208
50
- helm/benchmark/augmentations/contraction_expansion_perturbation.py,sha256=yni1UR2fviN0Wig8MpOp0zzLn4H-gYocTjKTpxBwywg,4850
51
- helm/benchmark/augmentations/contrast_sets_perturbation.py,sha256=OvuntCLHYZoWHXO1JTixWmnZD5iQDNy_yYrLwgAO15M,3338
52
- helm/benchmark/augmentations/correct_to_misspelling.json,sha256=L44RiJXlJCa6zQzTLf0MFHCOhFyRDRKfLQNXH-n3XIs,213429
53
- helm/benchmark/augmentations/data_augmenter.py,sha256=57LA6h7z1tVMy_xGcW46F1KRi3D4wnv0fi8XeJjsi2c,3849
54
- helm/benchmark/augmentations/dialect_perturbation.py,sha256=4DJB2gDj-JzhYtu-uxh9zgebChRDHpwMgKhju8OqP44,6317
55
- helm/benchmark/augmentations/extra_space_perturbation.py,sha256=D7y7oA4LwKJnVgTuLZOF3nQjmYRtkezuxY75qVQkD8I,843
56
- helm/benchmark/augmentations/filler_words_perturbation.py,sha256=NihLbpyxlXnncxM635pIyQ5LJvu4uPb4oKiLb8UU5dQ,2922
57
- helm/benchmark/augmentations/gender_perturbation.py,sha256=XGkEM-PnlH7aRWdz4zPkJ37j0b2XKhTFa_UlRl6bmKA,9389
58
- helm/benchmark/augmentations/lowercase_perturbation.py,sha256=e-lhkuKOV5QR_GnOp71vr2lHI97tiWSZ_jz57jcFbvg,519
59
- helm/benchmark/augmentations/mild_mix_perturbation.py,sha256=q27-c8_di1jaLhUuo5LzqOStnJcWIjSJ0a8Sr3S5TOs,1787
60
- helm/benchmark/augmentations/misspelling_perturbation.py,sha256=IE45qEE0HuIln62tAGvCP9B_RjNMJ5Xu06goL6b0fWQ,2144
61
- helm/benchmark/augmentations/person_name_perturbation.py,sha256=AIF5VVjNJ1OlUyCC1bt2dcP1KDn4gdgqd7Y2KGvHPqs,14422
62
- helm/benchmark/augmentations/perturbation.py,sha256=6JTCpKbV_PLcvLHkZT9OUFn4WnTMfIWZZazbKGLHdMA,2905
63
- helm/benchmark/augmentations/perturbation_description.py,sha256=VU3lo-7QJsHZNHKzQjYRRqxF73xUQ8AhQAaJUJk7x4Y,1185
64
- helm/benchmark/augmentations/space_perturbation.py,sha256=g4rbyoureBaOVf_lrRXIWYlL6YZpqbCOJk554iOCuIs,935
65
- helm/benchmark/augmentations/synonym_perturbation.py,sha256=komOV5M342_8unopnwN6gkPWpJIZXidywiu6PO9_riU,4151
66
- helm/benchmark/augmentations/test_perturbation.py,sha256=SNvAdUs2ZHP6OQAeOPMnvJgHldzdB2lVAS25SOU3rsc,11537
67
- helm/benchmark/augmentations/typos_perturbation.py,sha256=_F9zwvrLie8hX7mzUtQmYq6oq6yqaFiKGsvc9LAuBr4,2798
68
- helm/benchmark/data_overlap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
69
- helm/benchmark/data_overlap/data_overlap_spec.py,sha256=aj_l1l0qxUbUMrSWr70-Sb1j_JN-7WYop5BXPG_xj44,1998
70
- helm/benchmark/data_overlap/export_scenario_text.py,sha256=OiFsU_qME2_I87yDSNPfuAVI5Y9MbPEcEliYK6TaAEE,4527
71
- helm/benchmark/data_overlap/light_scenario.py,sha256=UFUr8plZD32e2TJTjFZLxTwD-ZRb9eYnHk2U3ZD8P40,1074
72
- helm/benchmark/efficiency_data/inference_denoised_runtimes.json,sha256=ios_dt-_8wtXvkVAx0iI2zwCxqHvk3XKTx31qHPalsI,4203
73
- helm/benchmark/efficiency_data/inference_idealized_runtimes.json,sha256=5w7reeZc0yc4cjH8kJGxQQSoe8yaRVX2SSlSrx0QWFQ,12348
74
- helm/benchmark/efficiency_data/training_efficiency.json,sha256=aH2moiBLStOLVi8Ci2KTK5ZkWlTBLK-B3fRfNZwhoSg,9763
75
- helm/benchmark/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
76
- helm/benchmark/metrics/basic_metrics.py,sha256=523j_t0UuuEGpIEyuk0u1TyQ2kY-vwwiwjg6TP5OkYM,43762
77
- helm/benchmark/metrics/bbq_metrics.py,sha256=H44mwKXLJ0PXo-sVKgRHCEKjZGnCahDD6GOQMWpnbOQ,6061
78
- helm/benchmark/metrics/bias_metrics.py,sha256=uVJFQvDSzvPR1ELu0FNYyExjRy2ThaJRCw8beEMDqJs,11309
79
- helm/benchmark/metrics/bias_word_lists.py,sha256=mx5JjW3mHffXIqo4GcQN-zENUEttBqQnEjPTz3J3J_4,13909
80
- helm/benchmark/metrics/classification_metrics.py,sha256=pgY8BToHw61lfPTswEW2Da8L4rkWJhthMBYVWnngzuw,5442
81
- helm/benchmark/metrics/cleva_accuracy_metrics.py,sha256=nSrtxGtIXPFcLrz9mNMYRfJqgQMzoCuf4i3TcQx6Rzw,2580
82
- helm/benchmark/metrics/cleva_harms_metrics.py,sha256=MrsSAvpOuGGFtVL8RfrG5bNbVTr3LmfhuNfnV4PrAuE,11085
83
- helm/benchmark/metrics/cleva_metrics_helper.py,sha256=8UwiGhekUmp7DxYWU4rxqX2v3ewkg-O5-jOh49iOGmc,304
84
- helm/benchmark/metrics/code_metrics.py,sha256=e0aqLcxBAdCc0qAqebzK40Ilv2Py6xZbosud5v169x8,5121
85
- helm/benchmark/metrics/code_metrics_helper.py,sha256=byyuI1lJgbIDPVJzywaBsam9zFMPPyn28g1grsK9xyA,22336
86
- helm/benchmark/metrics/copyright_metrics.py,sha256=X9j3YsfzWEoGpgPpIvCzm18-JggLAW5QFooifE1KqaM,7729
87
- helm/benchmark/metrics/disinformation_metrics.py,sha256=UVipWjmqiK3VONVTspCvAY9gTH8wE84eAJ1JRUT2iKA,10777
88
- helm/benchmark/metrics/dry_run_metrics.py,sha256=I6NfrWW9WsmFCoqCRIW_nSiFQW0gnyRl2_9JkIEqpNM,3656
89
- helm/benchmark/metrics/instruction_following_critique_metrics.py,sha256=-cHgmSKy-Z7ZT4ZFguI4EY6OiEp4MltXBq8zTEXBtaY,9757
90
- helm/benchmark/metrics/metric.py,sha256=GEa8xt7SSIaoZMs76bImII1HUxs9mhchIiNFaoG8IIk,18767
91
- helm/benchmark/metrics/metric_name.py,sha256=hk5WQ6uj_9EjgKKFawPenL2-XOMf-aKvNRkOxlu4nCo,1355
92
- helm/benchmark/metrics/metric_service.py,sha256=W3BqPoWikPQ8eGTUmcDIqMQK1IM1tymrjL741G2BSjY,989
93
- helm/benchmark/metrics/numeracy_metrics.py,sha256=panMWD3a1NPerg3Ix7l6NhR7jGOIQOQV9i_KysBeDA8,2818
94
- helm/benchmark/metrics/paraphrase_generation_metrics.py,sha256=EbdVAJkK9ueA6k4rEdB34jbE_xeh-o164sPiWieLAYo,1885
95
- helm/benchmark/metrics/ranking_metrics.py,sha256=oZgTHlCjjQ6imzrddPMZteTmQjG9QqLUxpiJzyqGREw,17341
96
- helm/benchmark/metrics/statistic.py,sha256=FuxNxMtAfiCkOxBS9KHlhEyxe61e0YXt2emvsufgPZQ,3424
97
- helm/benchmark/metrics/summarization_critique_metrics.py,sha256=Lf7PDuce62HDzyofsyxaOvH0QvzcaS-vJvDWtIs8xKk,4694
98
- helm/benchmark/metrics/summarization_metrics.py,sha256=gZ2MpzX0kQstRUnqzJjqTJ6z9h6kp4v_qhW1ahSIMVQ,16420
99
- helm/benchmark/metrics/test_basic_metrics.py,sha256=zbTTPxTKGLjGXQHyydar4YAd0JDZsPzju8J6ELGndOc,1076
100
- helm/benchmark/metrics/test_bias_metrics.py,sha256=N22a5UKqSd7TfYBrZS6ciFSOIc1s85pYMwk7m9ufpG0,6274
101
- helm/benchmark/metrics/test_classification_metrics.py,sha256=0snnCDOYoJqOb7xyQBgSd9bip19DB-WXRXR-Dcbb6QA,5550
102
- helm/benchmark/metrics/test_metric.py,sha256=S7LGHNCHuhMk582eHylw1tOasUBEf_7F0T4u3tey7b4,757
103
- helm/benchmark/metrics/test_numeracy_metrics.py,sha256=ls1ZIHDePKpHMoqAbf4HmJ1SIBjLFuLIzGbfg6OiZvM,4162
104
- helm/benchmark/metrics/test_statistic.py,sha256=AejuYLSeUwEOqpEMRKZFjnxu4HKUraeExU8TPmZEqW4,1229
105
- helm/benchmark/metrics/toxicity_metrics.py,sha256=6OsuOYX57m6Lvnpe6iYmJAjTGofQ73LBbyCtFCQ2Cbs,4033
106
- helm/benchmark/metrics/summac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
107
- helm/benchmark/metrics/summac/model_summac.py,sha256=HNWB5-C0Xnqml4TnXyPZdVegNu4hHJsY2IkzX3rm5Rg,17414
108
- helm/benchmark/metrics/summac/utils_misc.py,sha256=7_Q1c72cKt8PWtxn8u4R8nB53HK6_JF2nP8bBXYNk-A,1485
109
- helm/benchmark/metrics/tokens/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
110
- helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py,sha256=WeNP4yiM4TVrD9Kid-uVRmWIVDqETnBsMycZmIBiTZ0,665
111
- helm/benchmark/metrics/tokens/auto_token_cost_estimator.py,sha256=MAVCme5cK_ryPjoA6cQPvfpKwqdSfnliTaxwT0vi4R4,1974
112
- helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py,sha256=5igmDhWu7H8-qiJnAQqCTQd1WXRewMrpdJdo4qhvSWc,523
113
- helm/benchmark/metrics/tokens/free_token_cost_estimator.py,sha256=G_6UK6Js_NZ_eqY0ZQnrC9QJVMERGhV1f6v7xq2lM-Y,461
114
- helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py,sha256=vcIy78rZfjrgYksIWojDckf5Op24yIKkmeTXfhFf6mQ,1186
115
- helm/benchmark/metrics/tokens/openai_token_cost_estimator.py,sha256=7jgjcgmbcVfLA_nTOrWXKIF8TEXng_KnE6cSgsSXWmE,1398
116
- helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py,sha256=l9UQZ0aAIhCYuFbIjU3j3A3XXoMvwUvz1kvRtlDbtOo,1079
117
- helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py,sha256=_wJ3E3LbJB9XPLixTH82BYQbp32o3oij6Sz3lsZL30E,2648
118
- helm/benchmark/metrics/tokens/token_cost_estimator.py,sha256=fTGUfhHV6yMwpTkCEMTGMxKO8jskqJz4sAtwXT6M_C8,425
119
- helm/benchmark/presentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
120
- helm/benchmark/presentation/contamination.py,sha256=PiIdcaD3-xfExjOmyL5q4Ao2ASa-OlScJAB9u1Zxe7o,2811
121
- helm/benchmark/presentation/create_plots.py,sha256=lKD42IWdIFnYvlionF50erc94-wJQRZxFqqAfT4jxtk,28715
122
- helm/benchmark/presentation/run_display.py,sha256=9d_zOFK7-MCSqjRPTVnOHU8GATznyeXarS7YZRF1r4s,11232
123
- helm/benchmark/presentation/run_entry.py,sha256=J1QgLOP99N7N4bs7nzXWxyU3pOd-a1j8xwL9ag1nP_Y,1158
124
- helm/benchmark/presentation/schema.py,sha256=dzYMP9FFiY_3BkeAvz-i3iSeeIXoaSrnqh6l0ggc9cw,9634
125
- helm/benchmark/presentation/summarize.py,sha256=tzE--9WQ9DJi9tn3oQ_aIR4_wPT1giRP-1IPucyhlIA,65041
126
- helm/benchmark/presentation/table.py,sha256=VzVMwsgP3kItAM6FPRUaTphzJ-ZjriiuFbWlO1rJUMU,2879
127
- helm/benchmark/presentation/test_contamination.py,sha256=vEzr_CnNAWM5BUBe1yuc57uiRQ_bDNKPFJ059O1Wklc,517
128
- helm/benchmark/presentation/test_create_plots.py,sha256=5PPPegMTdBZurxyyUxI4rN13AVsjV3eQrwFqlobJ8UA,1286
129
- helm/benchmark/presentation/test_run_entry.py,sha256=r7jOIvNmGLyiZFFfkXGgDE6_SzhQ3Y7hLgmbzRPONWo,793
130
- helm/benchmark/presentation/test_summarize.py,sha256=pm9XbFsXfPEjWdx8cggC7CSDt4XDQoB5kGYyp5mMlNs,1629
131
- helm/benchmark/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
132
- helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py,sha256=Wyt7J5BAvAqC5JTqCW4fh7ex9-itX11P_9rLTocqvtk,4973
133
- helm/benchmark/scenarios/babi_qa_scenario.py,sha256=S1tPQY2x1I3hQL1JQ6wvUwvKyiSe7SqpRSW6N3_T0mo,5043
134
- helm/benchmark/scenarios/bbq_scenario.py,sha256=AMUeFLaO2jiB-FnBbe_a5sZ8OXBuG7_3lgld3dn_jtE,9581
135
- helm/benchmark/scenarios/big_bench_scenario.py,sha256=bSk8Ia4u_6OqMjiyadpYQAWN-8GFWqvd3Ft3JiVGpi8,8081
136
- helm/benchmark/scenarios/blimp_scenario.py,sha256=o1MDcHT14KFDET4K9otx8pDiIgXrhsD19pvO0mR2ADU,6260
137
- helm/benchmark/scenarios/bold_scenario.py,sha256=-Gg9C7lY8VFXU-sv5RmNOUZ3NXOJqvZtWsZC2iBEAlo,4110
138
- helm/benchmark/scenarios/boolq_scenario.py,sha256=rvSp5SwXMCVzBo5BFxfhj1Xv06_ksqKrtTQR7nPiS-o,8013
139
- helm/benchmark/scenarios/civil_comments_scenario.py,sha256=VO5G-cQ9qctmBN0O76uSewnO_mFslMo5mbR2ZTrjuds,4851
140
- helm/benchmark/scenarios/cleva_scenario.py,sha256=xhwZ616iz0CN3fYIfrXHcV1XlcRQjyPSzML8fq8D3l4,57939
141
- helm/benchmark/scenarios/code_scenario.py,sha256=BDjKQbFbV7btd_pRjUD0YA06zCnqqXCi8ekxjE0xws4,12170
142
- helm/benchmark/scenarios/code_scenario_apps_pinned_file_order.py,sha256=KC-5MQ-d8Nn46aDN4FaPxmd6yk1DtVUmVR-CIZsNCp4,1738
143
- helm/benchmark/scenarios/code_scenario_helper.py,sha256=TnXAlY-wdAFwIDylFItf0z7HOu93WD6dNThwzZYe330,5904
144
- helm/benchmark/scenarios/commonsense_scenario.py,sha256=Wgry4P4W0hHmeBrN61u5h6VTjET5TZI0oozC2ri3nbQ,9487
145
- helm/benchmark/scenarios/copyright_scenario.py,sha256=qw-ijottZ28DjMqMWSZzL9_FuRDAHf0YmGV27YstmVI,3673
146
- helm/benchmark/scenarios/covid_dialog_scenario.py,sha256=LqNtfb9XQestluN0Bz7yi-Zo0VRkLjl4F9IOYzhPw3I,3971
147
- helm/benchmark/scenarios/custom_mcqa_scenario.py,sha256=aZkt1Ums-kIVpLfd4Pfhe7Rf_fZLxjO2ANLL2aN2qVM,1915
148
- helm/benchmark/scenarios/dialogue_scenarios.py,sha256=MyF8CWkd_L42_gGLqG19YjCvpSpChCHeTj4jMUn_IkI,5630
149
- helm/benchmark/scenarios/disinformation_scenario.py,sha256=kQi0MVVoSDhx2vOTnUaCIttPXMf8zz7Eld2FD_77tnA,8504
150
- helm/benchmark/scenarios/dyck_language_scenario.py,sha256=vMxND9wPJenrGlCLhSw5UxOw3TV2Jq8cTmIXGpzEWaA,9318
151
- helm/benchmark/scenarios/entity_data_imputation_scenario.py,sha256=n2mnkmSeTznEy7S-GVumqpD9bt27yctbuEmtgQrG-Y0,6399
152
- helm/benchmark/scenarios/entity_matching_scenario.py,sha256=YjBX61TlL3CDQ3X6D-JyR-qlOYGLdoRXJxl9AEeqxYs,7022
153
- helm/benchmark/scenarios/entity_matching_scenario_fixed_random_state.py,sha256=TklbX7Kx4y-estV-YHUbI5O08q2qCZRrOmX9D3gZS9c,2193
154
- helm/benchmark/scenarios/grammar.py,sha256=Pb9vEP_0Ki87UdQCj1ym7QWJ24M4DRP6TXB5d3GnhLs,5597
155
- helm/benchmark/scenarios/grammar_scenario.py,sha256=bl-Cm9caDs077zSu38mzaS9maZ2gM-QazgjOEMFvxYg,1454
156
- helm/benchmark/scenarios/gsm_scenario.py,sha256=9fV2SEw3ocKNAD-TrDZZTpq4l7mbttQQWbO0YNz4e6k,2613
157
- helm/benchmark/scenarios/ice_scenario.py,sha256=vvk11cFPGUhg_CcGh3wEfVsGzrvMFgkByN-xcF-OOjI,16473
158
- helm/benchmark/scenarios/ice_scenario_pinned_file_order.py,sha256=fuirubIdi-rkJMfSd7YoDdBX2q0f5K7GGTN4XVapAUY,1613
159
- helm/benchmark/scenarios/imdb_scenario.py,sha256=-bfjm_43j0lsfGskz2THNeKABCld90rFTZ5yWt8FTHE,6175
160
- helm/benchmark/scenarios/imdb_scenario_pinned_file_order.py,sha256=fjW0Gkzg2Y3IAbtYJ3KC7MueWd9U8h0tlcBCqxYmRrM,1621
161
- helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py,sha256=PuwcuHnx3nCuRYEEzEQ6aRI9i7QAHtfg4DmLAG4M6AM,2205
162
- helm/benchmark/scenarios/koala_scenario.py,sha256=ihx39W1C9KyOh5MkdMqw1twUwzA8S5QPsBKeQRZcZGM,1365
163
- helm/benchmark/scenarios/legal_summarization_scenario.py,sha256=DTYYsf-74NFjHvrNa5ZWSgMywq9ss16r5eFZI2-xZJk,8644
164
- helm/benchmark/scenarios/legal_support_scenario.py,sha256=hu8SlDGrU6oUTQjnrLEagjFNniW96wMahKjnrtuyjzo,3925
165
- helm/benchmark/scenarios/legalbench_scenario.py,sha256=55D09TaELT_B__ERyyWUp92gGwXifNgX1eIOzw2pgBE,4398
166
- helm/benchmark/scenarios/lex_glue_scenario.py,sha256=vsy8Y03hoCwTJFgBdQf0koxlHDRFm2V9wT04n2K1tlk,10268
167
- helm/benchmark/scenarios/lextreme_scenario.py,sha256=HEpXLdLLywo6yfXaGF3zRgx1PiGPE2ILnpKYs9Dmsow,20396
168
- helm/benchmark/scenarios/lsat_qa_scenario.py,sha256=Ui68mi05N3WxGMZfCznJBscKkqaJ0ZEUO3PhIiD1xNE,6129
169
- helm/benchmark/scenarios/math_scenario.py,sha256=6HZ6y1kjE64tywZIee0ocVj-tqxXyvR6VfBldn7vftY,13849
170
- helm/benchmark/scenarios/me_q_sum_scenario.py,sha256=WLfED6DiIOUPd4PowydPmrQ4v26OsjG4iHumq7ka9do,4001
171
- helm/benchmark/scenarios/med_dialog_scenario.py,sha256=f7hTcEpxGfoxJ-WXNrSRrgVfbyuYpB624HbyRuMP-KI,7295
172
- helm/benchmark/scenarios/med_mcqa_scenario.py,sha256=JdVQuzKM4TRyWT6c1zpsMvNk1RYHnKdy_fRM6W7PCLI,5035
173
- helm/benchmark/scenarios/med_paragraph_simplification_scenario.py,sha256=e9WRx41XKJI06l3ZGkZe_LKNx-qd4hoWopfDncrr3KE,7591
174
- helm/benchmark/scenarios/med_qa_scenario.py,sha256=R0iDMzu3U0TnYEOiyjnlOWHGFwhvOJBVwJ5-Hos2eP8,4417
175
- helm/benchmark/scenarios/mmlu_scenario.py,sha256=wZjoGjaZNERyqSDHgIsv8fIANyUUQZ1ExQAa4cPzmM8,3790
176
- helm/benchmark/scenarios/msmarco_scenario.py,sha256=5DTWSDuinXVomlUTYfwOijXLifCSeG4be9Tvb7SANQM,34026
177
- helm/benchmark/scenarios/narrativeqa_scenario.py,sha256=FIzd4CTS6ZTOI_TJ8aPlkXYNfqbTIs13ZUqAt7ie8U4,5608
178
- helm/benchmark/scenarios/natural_qa_scenario.py,sha256=khKCR8xVhRqJtZoAfGQ9_CV6REkfrYn5ahLpci6rUzk,12540
179
- helm/benchmark/scenarios/newsqa_scenario.py,sha256=g7bFXWCTvRjofmmUJhx3jmCOvcVwi_P9LSVrNofvMLg,7260
180
- helm/benchmark/scenarios/numeracy_scenario.py,sha256=FxJJztIs16oCvMak6QpcycQKFlSu6oG3HMY9QicsSVw,30823
181
- helm/benchmark/scenarios/open_assistant_scenario.py,sha256=PH8F8zqYXXakr1xttBtFmENGMGciIpyijK4RCkJol_0,5495
182
- helm/benchmark/scenarios/opinions_qa_scenario.py,sha256=QmcqzokfZxskHa2HoEcaJAxdxZ1iQh-Qp-u4BWqzj2A,7375
183
- helm/benchmark/scenarios/pubmed_qa_scenario.py,sha256=zVL1gb3eVz-LbK2hfdnRR9ItaMSPlAGJorByWLt-4wk,7506
184
- helm/benchmark/scenarios/quac_scenario.py,sha256=SRAhMp6TAsmTRq6VRONLl3SEayFIe23He_mBhzkZ7qM,6628
185
- helm/benchmark/scenarios/raft_scenario.py,sha256=_5QhHS3opxxML7Rek6F-q5NVOf0M2UgbC6OTnQZ4C1U,4452
186
- helm/benchmark/scenarios/real_toxicity_prompts_scenario.py,sha256=GkgJo_13MWQQQTZbhlknvTR6ZrYr7NEn1WdMZrPs4y4,2400
187
- helm/benchmark/scenarios/scenario.py,sha256=do_ZO-vRnFREHosrloKDt-vFpGK0Qnml8wZc1eLKkVk,8091
188
- helm/benchmark/scenarios/self_instruct_scenario.py,sha256=jZ2MksT4N_4g_sp5egw7ycrsM-Ya786_RFmiYYdMvG8,2285
189
- helm/benchmark/scenarios/simple_scenarios.py,sha256=TbdQoeIxQAfcUOrFEGgnFrnSswzIuLPcek5RRV54GTs,1965
190
- helm/benchmark/scenarios/summarization_scenario.py,sha256=MlNMgsY369DC04nhMUdG2o9Ydi6yze1fGOjC0bK-UwQ,6847
191
- helm/benchmark/scenarios/synthetic_efficiency_scenario.py,sha256=pzifpsJJbucmTjujNqQnwQa4Y7wpQjkS6QjNXOrgTAQ,3096
192
- helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py,sha256=1b3e3WpFMNBV3li17-0Ug6QCSKO4qRFaWDF23bYNsvQ,16326
193
- helm/benchmark/scenarios/synthetic_reasoning_scenario.py,sha256=k8IGK6VABOr6wuha4HynP47peoAkmIViAVhScOtCANo,8345
194
- helm/benchmark/scenarios/test_grammar.py,sha256=sPlA36sHpThbXgnGlXyOuqHfDPe2epIafmzIeL0nkoU,1364
195
- helm/benchmark/scenarios/test_math_scenario.py,sha256=89rAt6GOAE5gBCWD5HXtCUEMcInKze-F-Is5r5LXFCo,725
196
- helm/benchmark/scenarios/test_scenario.py,sha256=OBcsrF7yCAbd3LNYo2pwFExZmixtdAyWaKRax5ngxyU,1870
197
- helm/benchmark/scenarios/the_pile_scenario.py,sha256=RqU8yXQJ4FkmEc6rO9J3QMXenyUZrsEZlLAQUx4-Wnk,4995
198
- helm/benchmark/scenarios/truthful_qa_scenario.py,sha256=iqL-tuqUQZjF9-DKAnI6wV-oLGC_I3aFuofdtJYHt8E,6035
199
- helm/benchmark/scenarios/twitter_aae_scenario.py,sha256=CRlPxVfkg3HPZV-lUMyCUSFOiAqg5IIPt-dq3qR9LU4,2096
200
- helm/benchmark/scenarios/verifiability_judgment_scenario.py,sha256=0buyJWLhy6edykqEiLrz0pZz-lA5u1UpqYny5aPuDO4,6011
201
- helm/benchmark/scenarios/vicuna_scenario.py,sha256=5z82_vz3I5pbWctDzC4kKHmtattkalSWQFURB1APYwg,1659
202
- helm/benchmark/scenarios/wikifact_scenario.py,sha256=cOVKgDisBdjPcmVMCLhTekdgX3hpDJUT-aKbvRSaMoM,5791
203
- helm/benchmark/scenarios/wikitext_103_scenario.py,sha256=PAPkmZdC4aIBQ1k29dDvTFBEFaPV1ZR1Ifif4FHoZqs,3087
204
- helm/benchmark/scenarios/wmt_14_scenario.py,sha256=Rsx3RpY7NYTS3Oz1PuGSaJLnDQoXTeSxWg2ddt8Q4ak,4474
205
- helm/benchmark/scenarios/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
206
- helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py,sha256=zrG5D-tJvPIxJRrMY3GDbko8UQAAr8CGV7660KoAl0w,4202
207
- helm/benchmark/scenarios/vision_language/vqa_scenario.py,sha256=w2acgyq0kFhh3iGkyCcHKAqWcEZvNEukCd0Nh5ay8TE,5111
208
- helm/benchmark/static/benchmarking.css,sha256=7PsUCff8YcoVxyWuALItfB4TZL55T7MbVQpoc9nZznc,2104
209
- helm/benchmark/static/benchmarking.js,sha256=hqRbqumir7DfnhWhT_bdGwo0bftd10YTONaGetsklpY,54529
210
- helm/benchmark/static/config.js,sha256=kIfkgr6gaMdFOAdqB35EvuBohq0DWYSQZbe_pTK09VM,103
211
- helm/benchmark/static/contamination.yaml,sha256=rAfh1DqwyUcDtyzHPQ2QiUK5eY7QfuuRtBXpZMn4TeA,3171
212
- helm/benchmark/static/general.js,sha256=qcsntanG5UMWK2vznSVAVFy9zd3BMc8DFfNa7KKezew,3053
213
- helm/benchmark/static/index.html,sha256=xIJGjMg0qn9eemfdBiNbTI0jzPfBD5x0v8HJF-dMqBc,3561
214
- helm/benchmark/static/info-icon.png,sha256=P-PW3Ek3NGiRAW5BXOjJRPBfMVqprjAqtQheGWu7zNI,3428
215
- helm/benchmark/static/json-urls.js,sha256=AaULgfHw8OLfrQLJpBHfcC013uavQnlNNFS9vzb0qOg,1981
216
- helm/benchmark/static/plot-captions.js,sha256=bTR8gYx-QqF_RJyKX-L-eQP7hSEtawfJSoADCvgjKag,3011
217
- helm/benchmark/static/schema_classic.yaml,sha256=S5Z3vMdWM9SGABDCWhQfQuXcuPS-f0BSjJjeTQbFDeU,151999
218
- helm/benchmark/static/schema_lite.yaml,sha256=r5XkUMX001FyguskqudMPjyCLjw01EJVT3DdelV3G8g,51318
219
- helm/benchmark/static/utils.js,sha256=bgN0PT53Dregc-nLmEmAEmg2psufWpS8jTf74WoypHw,7681
220
- helm/benchmark/static/images/crfm-logo.png,sha256=dDkauL_wJR_Luu7L7pltphS3a9HSLjDkpVLa6C9vcA4,62712
221
- helm/benchmark/static/images/helm-logo-simple.png,sha256=LtVAC4OgcWgMAob53rTrf7cRDu-O0z85ZOGGj9wR9hw,86133
222
- helm/benchmark/static/images/helm-logo.png,sha256=GTqbrxJr0oQXbBRq-8v6afY5zB5x0M6PhEbKRIX9qIE,280667
223
- helm/benchmark/static/images/language-model-helm.png,sha256=mG0-bkdziXeiF0wOGd67y2jnYmVKJYqhD2N5Q8VIF8Q,26563
224
- helm/benchmark/static/images/scenarios-by-metrics.png,sha256=F7g9mvIYopm-n7sDGg-7I0XCyZvloKsi2wIq1i6da_Q,51331
225
- helm/benchmark/static/images/taxonomy-scenarios.png,sha256=2MiuCLaxnuHvwsWWJHnZFc-rvoQIi_tNIjDatY7I-Dg,100766
226
- helm/benchmark/static/images/organizations/ai21.png,sha256=Drkew6Vlwi2_4_S8hjagK2x8smOwLKTNiXIT3rDiurs,10208
227
- helm/benchmark/static/images/organizations/anthropic.png,sha256=cNi8OdIshIIb8PdodcX8mAj-khaUD0O6nhah-_6nYfs,8017
228
- helm/benchmark/static/images/organizations/bigscience.png,sha256=fwQAwN1x2Fr_ztD_HZdcOkdFcyxuDjtS3B5-VuRNkuc,19036
229
- helm/benchmark/static/images/organizations/cohere.png,sha256=7cr4LI8WK9yPryQboyWK_T5baSND-d-tVrlPNflLQMg,8757
230
- helm/benchmark/static/images/organizations/eleutherai.png,sha256=uUURFF8YWY85mwGoKVEjArO5DUBCy4es5naCXsBzn6c,4526
231
- helm/benchmark/static/images/organizations/google.png,sha256=BtmXrVQZHr3WH5c8c23ent2FO8aPWeNwO8czl22lDCo,4914
232
- helm/benchmark/static/images/organizations/meta.png,sha256=VYDp8arkAe2eYRJhAOcIAsZY1qY0hqyOEQDgVMbX9M8,4646
233
- helm/benchmark/static/images/organizations/microsoft.png,sha256=9e5QFl23yTbnAk8u7lZKaQOf4oPHbr_aiQda5n4MZqE,50850
234
- helm/benchmark/static/images/organizations/nvidia.png,sha256=hvp1wZMwYxkfrVMvJs73PX71JwY5L8ZvxIH_fL4n6Po,27945
235
- helm/benchmark/static/images/organizations/openai.png,sha256=P4ZT5ISIlt6Dl0mOp7juSM4Y7dfyRNPqdc0PJuwNoqg,16877
236
- helm/benchmark/static/images/organizations/together.png,sha256=pmWjW4r7GnlKqFhKLPTiBeILiOighL3XzcSCsxWtB7U,48053
237
- helm/benchmark/static/images/organizations/tsinghua-keg.png,sha256=l9SzlZCsLF18BY876wYJcVgiQbgvwte7uoILPDcVwHk,7776
238
- helm/benchmark/static/images/organizations/yandex.png,sha256=OOCdcKubAP4x7h4VW7z5a-AHPWBiSDTjsIJea6ZiovA,27964
239
- helm/benchmark/window_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
240
- helm/benchmark/window_services/ai21_window_service.py,sha256=aV8uHeAc5_DmbBnh4tgLEgcC_QxdkPotl8Asyr2K1KA,12826
241
- helm/benchmark/window_services/cohere_window_service.py,sha256=dp4KTmbwxdHwxy7Eogfr_p0_F5gayFR86PNhUHaqb1M,7091
242
- helm/benchmark/window_services/default_window_service.py,sha256=TTVs6MXmyHQaZESd_J0DoYpDgCeTm9HIJQfBF0UlctI,1595
243
- helm/benchmark/window_services/encoder_decoder_window_service.py,sha256=Dg_rD0RtKIALtpvT4Wi4Am3zgFcIvgxfzdglw5fbbTU,2478
244
- helm/benchmark/window_services/flan_t5_window_service.py,sha256=39IZX89_tay3bpSGVoWDoekmhW-RUNATQuT-bNyFRTs,915
245
- helm/benchmark/window_services/gpt2_window_service.py,sha256=d-ys9FHoVI1u1GZOH734JPAOc8W6QZb1N3ZjooKmwz8,990
246
- helm/benchmark/window_services/huggingface_window_service.py,sha256=n6x2yWSUSO7bTlQqne4MbPhmZDLuN5KF8TkdTVVfxvM,2357
247
- helm/benchmark/window_services/ice_window_service.py,sha256=5z52rP-xAF_jckIPoogyoNFW6FQXqaq3SHybCaBuRn0,2005
248
- helm/benchmark/window_services/local_window_service.py,sha256=9nGiWMvmmqmuky3fe7rAAae9vGQYiD1HlSCtSUXk37c,4547
249
- helm/benchmark/window_services/no_decoding_window_service.py,sha256=s_i_cqIuU9p0GDRIBApaOHzjH7gHrBPTJ2X5NEcN33Y,1375
250
- helm/benchmark/window_services/t0pp_window_service.py,sha256=oa1vJRiyFPbkTb8eYnfjZNyKFJSvceipOq0U3Ys5e04,1196
251
- helm/benchmark/window_services/t511b_window_service.py,sha256=8CvkSfuG_Bg17gAEre_bmM4tUwoi9fAWpPxx6qCJwAE,1005
252
- helm/benchmark/window_services/test_ai21_window_service.py,sha256=HkpNSaJAClZfaa-bQZ2BrRm1UB_u4sLAGSBlGQqRUD4,8221
253
- helm/benchmark/window_services/test_anthropic_window_service.py,sha256=Zo3IHsKtE2yf3E29_FgBd_dNbwtgTO6LPArzXYEwI-8,4092
254
- helm/benchmark/window_services/test_bloom_window_service.py,sha256=fG1oJdfErc0m-r-x7WalQ4mW49R76Wt2F_bUuYDSEb0,4184
255
- helm/benchmark/window_services/test_cohere_window_service.py,sha256=j36eIUywlVqDyDnDlgKA4HgT8YE-ItmvOqCgaVk3csM,3240
256
- helm/benchmark/window_services/test_cohere_window_service_utils.py,sha256=sf25f9MeXzoqsbDzZ7d7le13hm8RkDe54nhLtKF2pqo,158150
257
- helm/benchmark/window_services/test_flan_t5_window_service.py,sha256=xv_EXbiRklveJPQtThYCSYF7qBYwjL7K4wH3Xu5z2Fg,591
258
- helm/benchmark/window_services/test_gpt2_window_service.py,sha256=3k25pLa_z__g4yoQL40DEXj-T4dGtrgif13N2NXs59U,2568
259
- helm/benchmark/window_services/test_gpt4_window_service.py,sha256=IR8JGLViwiFDmHOLgBfnPGeAJlV6IyRKt1BHOQ4P4X0,948
260
- helm/benchmark/window_services/test_gptj_window_service.py,sha256=sxsTpozKv9N-wZXtGl1prkQr9Md_q-tnCjO9zt226Co,2267
261
- helm/benchmark/window_services/test_gptneox_window_service.py,sha256=VwawHQnvrUKUr9Tl0eKjNLtZLEmpJPGxAUsAYMY1hlg,4103
262
- helm/benchmark/window_services/test_ice_window_service.py,sha256=hSJ52MQ2Edh6gTvgzCHFMvlZuidRhZF_t2E-RR4H_6A,23475
263
- helm/benchmark/window_services/test_mt_nlg_window_service.py,sha256=2PtD1orlVYYNa_JOV0_3bQPct3zvUx0YWzTnwwKZ74I,2212
264
- helm/benchmark/window_services/test_openai_window_service.py,sha256=q4ou-AK7-NvUkwAQP4ZuldrAgoGeDnfqM8Syj-7KZfA,2208
265
- helm/benchmark/window_services/test_opt_window_service.py,sha256=Fn_-q-yfh8wHhIdJ4ysTDmRkbz5S8I806NWLwRlTcW4,4108
266
- helm/benchmark/window_services/test_palmyra_window_service.py,sha256=WYSpFyr8Ll0-umMnqvUbIz69Vs4urqH4regmeC4sed0,4108
267
- helm/benchmark/window_services/test_t0pp_window_service.py,sha256=O99CgeJWEqfh5t6__jr_RrcYUvm2JplDbD3MpN9Ay4I,3894
268
- helm/benchmark/window_services/test_t511b_window_service.py,sha256=YtwycL2-oPLkWkZduSkm7AWGWzXJQC9A3sLjo7dy1-w,3897
269
- helm/benchmark/window_services/test_ul2_window_service.py,sha256=otiqcOPPFDgPzB2avNOvdmJwVUZH443J2KiQQ787xcI,3957
270
- helm/benchmark/window_services/test_utils.py,sha256=TMmPTkJ4vlFKv4vzSO35B6-0gnXbQ0nvNZJR6CZN6qo,3227
271
- helm/benchmark/window_services/test_yalm_window_service.py,sha256=NDvZyDkgO-eYjt1oEieAJiNFFgt9qqXAa38CqWaWq4c,4197
272
- helm/benchmark/window_services/tokenizer_service.py,sha256=dPsEhA1I4SUvsR0UBdzJ2wsFahw8GuF6qhE8CxiPZRY,1051
273
- helm/benchmark/window_services/ul2_window_service.py,sha256=R_VEzOb59zQE9mmbTLunQeIvLAtK3-97h-B2_oc0Uxs,1021
274
- helm/benchmark/window_services/wider_ai21_window_service.py,sha256=o52sBY0OgHEiPrMv5rve8FzE4uXy_QuDFdjTgqQpZM8,727
275
- helm/benchmark/window_services/window_service.py,sha256=YmMyFRBkQ1sH2bGPEuUBC7x4nt6n8wKRVWyENlr6B28,3361
276
- helm/benchmark/window_services/window_service_factory.py,sha256=SVyBM3nRITDc9fOk8KUfmXZ79hE-P3UBrFtPHFxLbYQ,3230
277
- helm/benchmark/window_services/yalm_window_service.py,sha256=tPOAiUsz3Z6aAU-nBq81lLDtYttZopRGJEcLNfCVBak,1813
278
- helm/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
279
- helm/common/authentication.py,sha256=RlMx29_TSrfU7ujE7dJkxmFub5EqLj2NswV5lAVFFDk,179
280
- helm/common/cache.py,sha256=uq95XlMjFRsQ9g6yCrfX4SE-daaVi59K14CcxGZPGmg,7199
281
- helm/common/cache_utils.py,sha256=sLv6YBu6Js2oGa_c2LXrch3bInu7scthQoMoWzbgTi4,522
282
- helm/common/codec.py,sha256=gTh6AwIQ0Bbul_QSnIO7eItwMZmYtnkIrG1jkc4GOL4,7100
283
- helm/common/concurrency.py,sha256=8THtHlCtXo5c8iCuz_UcBBdzZX6aiEALLc4u0M4SYL0,856
284
- helm/common/credentials_utils.py,sha256=O-57nUgkWLbZF0k3lsSaVGPPHj2_OYeVuCMe0to3bRE,1118
285
- helm/common/critique_request.py,sha256=Exu8Ans05zCU5d5-AglEbG40mBwKYED2Z3WqY_XjXBY,2772
286
- helm/common/general.py,sha256=nlZ6jRgdSNpM66BWEmjTkun8AXm0IaLyBZnrnnsLyk0,11679
287
- helm/common/gpu_utils.py,sha256=pmLq6ipYNLEm28VxxSNeZuVt-gAw-WnYmBvxP1P1p6M,480
288
- helm/common/hierarchical_logger.py,sha256=EnKLnfbQftca08EJfjGEQb4tcnCKbx-JtwLnoCnhMQs,2908
289
- helm/common/images_utils.py,sha256=Jp6nlqYfrBvtwuWdWThQvuSHWWw1qM2WQH7KJfxQ0VI,1514
290
- helm/common/key_value_store.py,sha256=cZPYMc4Fp3oIk3O908YCyIFIx-KVIBrVKUO0qeLf9WU,2333
291
- helm/common/media_object.py,sha256=nAl4phYCAl9TL1q61TlW4zwInDcdgwzYr2X4JZfFJSU,4709
292
- helm/common/mongo_key_value_store.py,sha256=Lzi7m5NeJeHBOGEZUMPQiaFxz_U88xzyXNdU6OQkAec,3796
293
- helm/common/object_spec.py,sha256=_usgTDQULBF6_jy7C6m-9ZNVvNxbGoTE_CdGcSvBASU,4327
294
- helm/common/optional_dependencies.py,sha256=Qam3QCHff8tuXbS-fCw-MVe-pK18gSvHw-uQoXXxT7M,616
295
- helm/common/perspective_api_request.py,sha256=WAVwtajNVmi5XJNsPcorGEAVrqkpPSk-Kd3b0hJghbA,2427
296
- helm/common/request.py,sha256=mWDXulEc_o6OQlrgvykGttWE_2XAXNnlGTRl1hCQFWc,8019
297
- helm/common/test_cache.py,sha256=XqboYHQAkFWIHPsuIjuageRSLeN7QoATKF7wwxggPqE,7054
298
- helm/common/test_codec.py,sha256=igL--k-2DwAy0eoMr8D9Xs8MOjBoT0LutbMPzDlTNkM,5885
299
- helm/common/test_general.py,sha256=zOxSwWNgWnWHsXKcG4NZ50GkWicn4uZ4jPVypSwFaQE,1672
300
- helm/common/test_media_object.py,sha256=AAm9DD7MC-ZvTwiqXA-e52U6L4S1noxItW8f7ARi6DY,1650
301
- helm/common/tokenization_request.py,sha256=fjEU2yi15vbf16tuoGEMEJ5vKpxfn_nWXwIstZ3M7gU,3546
302
- helm/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
303
- helm/config/model_deployments.yaml,sha256=a35_i5UbzfnbvyNBrgA4lWXVp4Bx_ORhyW23MYXZrSQ,46211
304
- helm/config/model_metadata.yaml,sha256=49aFr-2zWlyf9sjobOxZjChPd6xO6Ui0cFNPfnHZJCI,67906
305
- helm/config/tokenizer_configs.yaml,sha256=4I0gfNlOM0rrTnVnJ-ntAdw9hhcas18Nx1DvcTzFRSk,8731
306
- helm/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
307
- helm/proxy/accounts.py,sha256=AZ0hCCD3ygSXtgZtnFJFJE1jOSRh5gMtQSM8TeJzqn0,13533
308
- helm/proxy/cli.py,sha256=l8F7UYqrIOoBD9ZCIxJFA4fhxlzhae0-2Nn8A7FMkzk,8244
309
- helm/proxy/example_queries.py,sha256=qTUY41MK-i62TvqOIQCsb365-RdxMb6j5lVF3LNqX2g,4263
310
- helm/proxy/query.py,sha256=eftbiUICMh8QIHVs-7cLtv_rDXKeKdRPmwjLMu0TDxQ,645
311
- helm/proxy/retry.py,sha256=iLZmKATEJQa9jsSpOIx6YDRhmrA8G1Qm21cUxCuo2Ug,3490
312
- helm/proxy/server.py,sha256=iIfZdVYmfYcanseWIiaHRGJX5Wa5Oj0BVkanomvsCIQ,8020
313
- helm/proxy/test_retry.py,sha256=8h398auzjW9VnlTJWllxR-bdpub-XFp8EN8LWDEnEHM,1049
314
- helm/proxy/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
315
- helm/proxy/clients/ai21_client.py,sha256=-ju4YwPGtimP3Jp2q8YLFRZh3BlHGfhg0l00vDxWdwA,5865
316
- helm/proxy/clients/ai21_utils.py,sha256=mlg3h615kyckccGZv9rqsP4Y60O3XpwyE-UURRMrxII,471
317
- helm/proxy/clients/aleph_alpha_client.py,sha256=eCW0nXGFQevbm3YzJ9SM6Udo4N7D6Hra4KdB9OybG1g,3987
318
- helm/proxy/clients/anthropic_client.py,sha256=uUGIQcyH4mjmG6zuOrdyGCcZXoj937dyIJdOAIePcz8,23945
319
- helm/proxy/clients/auto_client.py,sha256=1UTOwsgpVI3gRSHVQZBGtbGGPPeb29sbzF_nB3qH8jo,9469
320
- helm/proxy/clients/client.py,sha256=rCvFnrPXggXvCv_iVoF8WEjRF_K7uqsQG1tDkiFcRKE,5753
321
- helm/proxy/clients/cohere_client.py,sha256=6JfFwWXnGB3e_iDJfAtrSz9fVy1VWz5PTl2LkDRs3gY,7565
322
- helm/proxy/clients/cohere_utils.py,sha256=aYmj60m0e9RF9BIdxp1vmA-uZv17TEALw0dbgTUSpCc,504
323
- helm/proxy/clients/google_client.py,sha256=cPcab3GBYIS4cMRLzgAAwYGR9uzN2ArAuvTeJRI3XiE,2906
324
- helm/proxy/clients/goose_ai_client.py,sha256=D3U0At_wiTDMttVbEo6b_x7QpBWM1zYhlZb2GruG-b8,3817
325
- helm/proxy/clients/http_model_client.py,sha256=efpoT2y5aDd-Fr5hTF_4j9buv6_4kv1wpbY2_mGAyiI,2826
326
- helm/proxy/clients/huggingface_client.py,sha256=mGBqVj-QfZ61cF89FFfPLaWwzqNqV1aoGmtmAaFq8QE,15061
327
- helm/proxy/clients/lit_gpt_client.py,sha256=FTcIU4oHAeAych6I68SHwYaNw4ngY7jzxLBRhjgyQZc,6214
328
- helm/proxy/clients/lit_gpt_generate.py,sha256=8DdBE9ReQ00NbV3KMFYc--PlO9X-HMOR0Rhm5CADWEA,3103
329
- helm/proxy/clients/megatron_client.py,sha256=hFJW_1PFvirYW3vUnooCUX3_xhx5nxbw58tx7_CmVrA,4122
330
- helm/proxy/clients/microsoft_client.py,sha256=1-vTybeAxz_mgaQZ4JBfVhiQVA2Zir-2o9ebM7Ze7KM,7838
331
- helm/proxy/clients/openai_client.py,sha256=w_ZRbMWknO4zyC5jfYM2hkXUvLBktDSLi4uk3lujAwo,9822
332
- helm/proxy/clients/palmyra_client.py,sha256=TNv1PXkzXj61TRgekHFJLdYyaXmjL1L2IYKUyBM_VHE,6565
333
- helm/proxy/clients/perspective_api_client.py,sha256=wwfD62GdxKsULZNXHuzx-_JUde96MFIBwZw5QGrVzc0,5880
334
- helm/proxy/clients/simple_client.py,sha256=uLa8fD56UV7EMs_F4CnpLwHXZD8vfijv_hFYzyM0dbI,2124
335
- helm/proxy/clients/test_auto_client.py,sha256=q_UWDZ4C34KKXT8rxmO-4ZR81KXqYUIcjrT6rxLG-Ns,3587
336
- helm/proxy/clients/test_client.py,sha256=Oqj53D5JC3-FdfOGHUdOvnOvklKTd_XVf7Tf7Xo8yno,1722
337
- helm/proxy/clients/test_huggingface_client.py,sha256=Eg1Qjr2uLScgUdlvpup-lTloH0YzXTlR74z_mgNnoNo,2613
338
- helm/proxy/clients/test_together_client.py,sha256=SDhvSTO4QhzQsd5l3b6Sn_XEABuUwalmw19cB6pOz14,3432
339
- helm/proxy/clients/together_client.py,sha256=JX1l2sVjVlVgwldHxhgvyxbVGo0-jP7ta-Q38XXP5kQ,15713
340
- helm/proxy/clients/toxicity_classifier_client.py,sha256=AI_FizxMurubTIyeceRdkixSnhWQbcD-oEEONj5ve7o,464
341
- helm/proxy/clients/vertexai_client.py,sha256=9jE7gVT0NfhICAe1nLbK1cQBWHncT8UJ6cFbNU0_jgQ,5195
342
- helm/proxy/clients/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
343
- helm/proxy/clients/vision_language/idefics_client.py,sha256=WJAAFE13n6qlj7YqCYJesozRtGq13a99qW_7la88gHQ,7252
344
- helm/proxy/critique/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
345
- helm/proxy/critique/critique_client.py,sha256=ATZuXw77lejwtpgLg3Soy3VDyv8D8xetl0o4guxDM2M,1764
346
- helm/proxy/critique/mechanical_turk_critique_client.py,sha256=OcppmFOMweBSfVTiLIICIwjvPpHHTkdu9fFUTaubitQ,574
347
- helm/proxy/critique/mechanical_turk_critique_exporter.py,sha256=taULrc_cIP0O9c5UpGz3l9DmWQadTVzN_v-qzTgMoyo,8470
348
- helm/proxy/critique/mechanical_turk_critique_importer.py,sha256=fhMf4O9O712GEQD2lLv7HQDtN2BARhOCrnDmlMYbwmw,5535
349
- helm/proxy/critique/mechanical_turk_utils.py,sha256=mKpUv4zz3s5ptzDY7UrwuI7Cr5HmNgSjPC10BnN9AL4,1766
350
- helm/proxy/critique/model_critique_client.py,sha256=VrZrYQIUur8NT-jA18lh8aJItP3jRJdHBx4euXY0Kk8,11144
351
- helm/proxy/critique/scale_critique_client.py,sha256=B4povtceyfal95eE3N7em9cC_B5Vy4jMrHXcsXc_5m4,15889
352
- helm/proxy/critique/surge_ai_critique_client.py,sha256=HnzgAoF4Du9Me0GS_lbNaozZslS4a2OZx735gh-coo0,8357
353
- helm/proxy/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
354
- helm/proxy/services/remote_service.py,sha256=YudZE8RWX4cB0YqYCC8yUPLrT_nfZmaSpD1tubhkT-Q,7395
355
- helm/proxy/services/server_service.py,sha256=doMe3U2AFS-Rm9xfBTct4BUMXenpXG9lY25ns-_FdcE,7931
356
- helm/proxy/services/service.py,sha256=pZicxz9eOQn-CmQ-EHrZ0Apfp-OZXjopiLW5XduWMdE,4900
357
- helm/proxy/services/test_remote_service.py,sha256=qZrDGDpvhxn3Ip3hjWo0kJ9POtSV2W3oQy67UhrlGSo,6527
358
- helm/proxy/services/test_service.py,sha256=IdKg_Qs64w9--HhQnbl5XnHZNI4F4tZ9YNzrCT8frjE,9553
359
- helm/proxy/token_counters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
360
- helm/proxy/token_counters/ai21_token_counter.py,sha256=Y0LmjyLarUIlshQIhcsvD-nUJQ1KmEAW-jJuinYAJrQ,962
361
- helm/proxy/token_counters/auto_token_counter.py,sha256=4lbYAsVwXMjbN1blac_6zgLTpSJaThQhYp_yw_TU_s8,1866
362
- helm/proxy/token_counters/cohere_token_counter.py,sha256=DHzZ0V3CnUvWvTdgCBjC-SEBbYxu_A7kfXsRdaFr0ro,488
363
- helm/proxy/token_counters/free_token_counter.py,sha256=DVdKgaQDxd42A8D9elcSunmtxAAtzUr1sEPRLJQ4ZSk,399
364
- helm/proxy/token_counters/gooseai_token_counter.py,sha256=cY5-3MJcTf0Wzpk5c3-8C8rXkEmdr4yBPv60SuBFtPI,1071
365
- helm/proxy/token_counters/openai_token_counter.py,sha256=ZIrKmogZZPFn_ay9HFZLRNK4s9YVh9Byxz64QkUQdOE,1117
366
- helm/proxy/token_counters/test_ai21_token_counter.py,sha256=rDRA_TNbH1rLpiBnsJH7sg1DFQxHg1-1ULwXvLZp3q8,5610
367
- helm/proxy/token_counters/test_openai_token_counter.py,sha256=k_HH3pP-fKxCIUDM59HLbpSWdvrDINj7j7JVN1QF9qY,5024
368
- helm/proxy/token_counters/token_counter.py,sha256=x8KyTR82EedgCQUuneQiVq9AiU1B3_CHPmKPNumClHc,429
369
- helm/proxy/tokenizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
370
- helm/proxy/tokenizers/ai21_tokenizer.py,sha256=snBAumRIGwhTFm3oeyVexpla7g6paeTIIIMr51f-S44,2238
371
- helm/proxy/tokenizers/aleph_alpha_tokenizer.py,sha256=UlWC_SjObBvexpZ3OfKZT2yjhbSsHlKjQe_oWuRrXno,3818
372
- helm/proxy/tokenizers/anthropic_tokenizer.py,sha256=15nsRyrm_sKWgrdR0sX95UDd5PGR8aAp-NjyZKbeaNU,2102
373
- helm/proxy/tokenizers/auto_tokenizer.py,sha256=1mG2TbznYuQikOALDHDf0Rq50Qf1o4s0t7-A4m0cFHY,4299
374
- helm/proxy/tokenizers/caching_tokenizer.py,sha256=kSegrCFotRevSDgJsn0g52dWiSUCNa7_EZpRNrELeUE,8163
375
- helm/proxy/tokenizers/cohere_tokenizer.py,sha256=9mRLdwJhy_JgUW-y6-CFc_iez2raky3S7UqPJ0dcCbE,3745
376
- helm/proxy/tokenizers/http_model_tokenizer.py,sha256=nfIANZt_0UCh6SM7VZI72wmBov80PkE38v5kgeVPD70,3062
377
- helm/proxy/tokenizers/huggingface_tokenizer.py,sha256=G2V4B7SwXS1Vsd5eytFV1xqgMVMzKFQyPHL4prIdbI4,8379
378
- helm/proxy/tokenizers/ice_tokenizer.py,sha256=4ZTIRpmt2cqwcxnmrDpCRhiJ0BI3ELE-GHoBuHWgrDA,1200
379
- helm/proxy/tokenizers/lit_gpt_tokenizer.py,sha256=LMrpaje64UmnDKoYjPG_RQeXVA4xQUwW5t48IJIeLaQ,1660
380
- helm/proxy/tokenizers/simple_tokenizer.py,sha256=GjQz0D1IQ3reXO5e6fkYkn7a9b_vmmIKch2wT13qljE,1134
381
- helm/proxy/tokenizers/test_anthropic_tokenizer.py,sha256=nN2YQMWtNCCFmETZA6OBGhh9NDoz6t0xnA5OLTHoTVg,3798
382
- helm/proxy/tokenizers/test_huggingface_tokenizer.py,sha256=4A7F-9eCWOlQJk8QRwzX1h37YZJf6_3b7DwXUBc7BPo,6041
383
- helm/proxy/tokenizers/test_ice_tokenizer.py,sha256=-xi_f8TBSkAYr5CcA56HDq7rZ9HAGd99J7twNfkLzFU,2619
384
- helm/proxy/tokenizers/test_yalm_tokenizer.py,sha256=qWpKnUuAlePd6t-UJB_mAiBwtAacnC8caKXLJ_GdTkk,2477
385
- helm/proxy/tokenizers/tiktoken_tokenizer.py,sha256=FU2g_FF0pVoyspYhHcz3SyCBGNbsTby-nWVrj0Cq4_c,1265
386
- helm/proxy/tokenizers/tokenizer.py,sha256=1zi9fhgrs2znINKDnoEhW-6UatsVj30hRjQbTDbOdZg,1583
387
- helm/proxy/tokenizers/vertexai_tokenizer.py,sha256=mzlc7ZJt1fRT2Ib82-Am-UALOBjjLSgmYbDG3VdhkdM,4111
388
- helm/proxy/tokenizers/yalm_tokenizer.py,sha256=uN7ZLiqG2uCmL1iWy6Y_Je6XuATyvqupWPjJDLnjdNk,1411
389
- helm/proxy/tokenizers/yalm_tokenizer_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
390
- helm/proxy/tokenizers/yalm_tokenizer_data/test_yalm_tokenizer.py,sha256=W9p5QNn1GSm-y85yVEQe_82zn5CVK_vR6jvhk7JTs_k,869
391
- helm/proxy/tokenizers/yalm_tokenizer_data/yalm_tokenizer.py,sha256=wXP-LmFLUjRfCExkNHbwVUINE8qFPwvv9h7Sf_pVNbw,6395
392
- crfm_helm-0.4.0.dist-info/LICENSE,sha256=bJiay7Nn5SHQ2n_4ZIT3AE0W1RGq4O7pxOApgBsaT64,11349
393
- crfm_helm-0.4.0.dist-info/METADATA,sha256=xHtXfsOCRLde00vrfKYVHJ9018nSXred3WxwFXUWkxw,13501
394
- crfm_helm-0.4.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
395
- crfm_helm-0.4.0.dist-info/entry_points.txt,sha256=AvH9soAH3uey9xffisWewd0yrmPWGASC036jHd1SFyg,300
396
- crfm_helm-0.4.0.dist-info/top_level.txt,sha256=s9yl-XmuTId6n_W_xRjCS99MHTwPXOlkKxmTr8xZUNY,5
397
- crfm_helm-0.4.0.dist-info/RECORD,,