crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

crfm_helm-0.3.0.dist-info/RECORD

@@ -1,396 +0,0 @@
-helm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/data_preprocessor.py,sha256=aNdM-o2t4qkLIQHiQeWUFg03DjjJ8HTBIphYCK8pXVo,2173
-helm/benchmark/executor.py,sha256=zi3YzO1rFHegwWYZcu7HC6ClqgqjhzSAdVgIOMk20G8,3586
-helm/benchmark/huggingface_registration.py,sha256=PYuPamPXPU00PgCGPH76wlvSd6jQegkFOB61oNBcWus,2644
-helm/benchmark/model_deployment_registry.py,sha256=8nJMaUD_9R-hVDrI8YiAQ1dmFywWYmfESRrjNtQPwkM,3230
-helm/benchmark/model_metadata_registry.py,sha256=73zAGuEPerNTMF_RxW8kjMfc91gREyoZ_GOe307_fMw,2192
-helm/benchmark/run.py,sha256=Ql0JsSmiGLzRXBgb7O7VLz0G_Wy-tLisYUJTDmR8Kvg,12109
-helm/benchmark/run_expander.py,sha256=-3VVv4gVUM9CNM71HRl85vS0Kl44QP79PnVAZ1b6PsI,37884
-helm/benchmark/run_specs.py,sha256=uOc9IIhoy0uXFDPhqjIagXp4E4UsqpQzRwk948dAYqk,97948
-helm/benchmark/runner.py,sha256=mRA5tcKKuvFzXLR51M49rF6pPPG7so3_6U-5HyVSjyg,12653
-helm/benchmark/server.py,sha256=aWlLkaeJV0e2Ge3w8wXRszAYzNXNXzqnnTyoJ91MeyE,3305
-helm/benchmark/slurm_jobs.py,sha256=uN-rdTcjv8Klm5xncovse4VN-RbkbCtUXP8hoA1BL0w,2903
-helm/benchmark/slurm_runner.py,sha256=hNZWhWkJx8PWqGIpXmoJLS3sprZZP6z-mdH99Q2_56A,15320
-helm/benchmark/test_data_preprocessor.py,sha256=xIwFuhb7WpdGLfFq9gfBEzPlsmyIEZMuHW4iPtq2HEY,2180
-helm/benchmark/test_run_expander.py,sha256=nVxlsu5mFC-omMTTSDR0CHAaDDnI1sXbLxjCj4Xuu6I,1161
-helm/benchmark/tokenizer_config_registry.py,sha256=BKDVqRK6fiw6Ob9Cefq4se5JgAd63Ftey7R_TredEA0,1624
-helm/benchmark/vlm_run_specs.py,sha256=2Jzz3fgQNxVjkOP7o7MjHIZVWButLveUotausqiIaYk,2223
-helm/benchmark/adaptation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/adaptation/adapter_spec.py,sha256=VDDbnoeevlz72pJ_ZPqNhITPpuxV1cgVlipzLb2wHlA,3026
-helm/benchmark/adaptation/prompt.py,sha256=MATerIUIhFp_BMGvK7bLpNtWH6Oi4kknjBjOkr2bHv4,1948
-helm/benchmark/adaptation/request_state.py,sha256=o3OpZbB0TJFiZ2Nmhvg3vWmByaUSYTffT_WnoNb7w68,2712
-helm/benchmark/adaptation/scenario_state.py,sha256=ZflBuNgvN0JqUhshFcy0kTweO1WJs6j5UCaTxWTMe0o,1747
-helm/benchmark/adaptation/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/adaptation/adapters/adapter.py,sha256=bKJPLT0FHUh2SQ_3S4g32ibwM8KHBfH04RJN8dj5l4Q,2899
-helm/benchmark/adaptation/adapters/adapter_factory.py,sha256=VUBOfOWKC8k0FimJ5ZW8CZZVK0W33eg801wK1ZhYIjg,2627
-helm/benchmark/adaptation/adapters/binary_ranking_adapter.py,sha256=hNw2KnVNnSn1jGM82Rf93enu7bXKhc-JxV8G-oOsHnI,5713
-helm/benchmark/adaptation/adapters/generation_adapter.py,sha256=qn67Xfm-vsjP2BSIDr0Y3sb8dSGBguzY9-MOiscDXXo,1994
-helm/benchmark/adaptation/adapters/in_context_learning_adapter.py,sha256=IyBk314PWm-6GEPq1iinapIOEwzQCX4zkT3C9GoYq5M,14185
-helm/benchmark/adaptation/adapters/language_modeling_adapter.py,sha256=Pw2_PlPFPzi17gXl6lL03T3Y9xYpb2gRMU_pD7bkNDU,14183
-helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py,sha256=VJ66MfIGQWJg0VXCV0MJEMwF9Jx1DeJ7RxsgYlOTx_4,1889
-helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py,sha256=7oRHPXdFLDiN8v6Nkw4FJJqiisJb8ULN_XWOtl3us3Q,3920
-helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py,sha256=VUUzM6Vqh9DHHOU3FqJ08rtng1x4yu0AX7SGWMjZj2A,2091
-helm/benchmark/adaptation/adapters/test_adapter.py,sha256=u-c7K6VH6yvymRtPjoWRBIjk0vs41g7IWOfv1w9Sxn0,620
-helm/benchmark/adaptation/adapters/test_generation_adapter.py,sha256=uJAchU2nE_w6QvsZGhM8itllhA5_lN-zrW8RPdHrBkk,11291
-helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py,sha256=-WR150g1n-bSEFMFnhdD4Hk8-DZmAzd7UHFX_EBRZ7M,6268
-helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py,sha256=gwdhdqQBwWQniZIH9qV9y1zgugjNRhL41EqfQ1X7-FQ,7060
-helm/benchmark/adaptation/adapters/multimodal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py,sha256=3RjCqvbUPJ_f1HAYCBLrfqI49u5wKRjHJznh4BXfe5k,1816
-helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py,sha256=0lZ5eS1jtc18ngQAIIFXWjnmdgNeg4xlGXbd8MlTNk8,6235
-helm/benchmark/adaptation/adapters/multimodal/multimodal_prompt.py,sha256=5jdVVfWf0lBErwzDamIHozZSbf1mEkpmACX8mPlHJc0,1907
-helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py,sha256=V-VAs0brUiPDNDSGe9e0CaAPczLlGjdb4fyzXhFnI6Y,9909
-helm/benchmark/adaptation/adapters/multimodal/test_multimodal_prompt.py,sha256=2OZhhOtEZ7RvItHZD4AGsefXvVUwEQZL2T-ESHPsvdI,3482
-helm/benchmark/augmentations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/augmentations/cleva_perturbation.py,sha256=W_HXYqkvrYIN8pc-uZTD5wMODfTZKPmdBg8IvjO0cl4,29452
-helm/benchmark/augmentations/contraction_expansion_perturbation.py,sha256=j6KqdLnqBlp8rC-n0KDbxx12E581Q6RoLXdjrutf_F8,4838
-helm/benchmark/augmentations/contrast_sets_perturbation.py,sha256=BzXL5FwGTFlYnHbqJOcNS_Dco_wKoU3CsCELXgBP9Pc,3530
-helm/benchmark/augmentations/correct_to_misspelling.json,sha256=L44RiJXlJCa6zQzTLf0MFHCOhFyRDRKfLQNXH-n3XIs,213429
-helm/benchmark/augmentations/data_augmenter.py,sha256=57LA6h7z1tVMy_xGcW46F1KRi3D4wnv0fi8XeJjsi2c,3849
-helm/benchmark/augmentations/dialect_perturbation.py,sha256=HJXQ87CSGt98QYIeIu24KSPN7NG8CTLxJsqolYeD7So,6309
-helm/benchmark/augmentations/extra_space_perturbation.py,sha256=9_pmthcyFfuYu6GsJB03hKkhvDZqqfH7hOeSNZRohSg,835
-helm/benchmark/augmentations/filler_words_perturbation.py,sha256=HWzGrH405RwxWxtgUs6Tc-yZZwSRqLHM6zOppPquDgI,2914
-helm/benchmark/augmentations/gender_perturbation.py,sha256=Q_Y1POwiCrM_cRNuzaTX7_anjO5U-o9dhT9fLQdvkOQ,9381
-helm/benchmark/augmentations/lowercase_perturbation.py,sha256=gh4h1WVEBIgd4LqZgdHQbYyWd4oMJoV_RiLyJTefxPc,511
-helm/benchmark/augmentations/mild_mix_perturbation.py,sha256=2ItUy3ck6BgcoJuqF6F_zdex4trDrYu96Auz8rt3ytY,1779
-helm/benchmark/augmentations/misspelling_perturbation.py,sha256=uETjdCot323yAeFm9a9DkL5uNa-h50hMm2HOhKYBjaM,2136
-helm/benchmark/augmentations/person_name_perturbation.py,sha256=gn0P6V_tlbC0zg37mUGKfDnPQmH-HQPsyAKxZ-8pa78,14708
-helm/benchmark/augmentations/perturbation.py,sha256=xwOu6aujOIAYINVuvWamh3yYB40-rCy5cnaHuhfSFTc,2744
-helm/benchmark/augmentations/perturbation_description.py,sha256=VU3lo-7QJsHZNHKzQjYRRqxF73xUQ8AhQAaJUJk7x4Y,1185
-helm/benchmark/augmentations/space_perturbation.py,sha256=7OdpoibdizoPDBPpLc1ENyJhhKW8oRNaUeR_j9-PtJ4,927
-helm/benchmark/augmentations/synonym_perturbation.py,sha256=f-t43JhVVX66tx3KqcVqav901wIj1O-XMhhlctqJydg,4143
-helm/benchmark/augmentations/test_perturbation.py,sha256=SNvAdUs2ZHP6OQAeOPMnvJgHldzdB2lVAS25SOU3rsc,11537
-helm/benchmark/augmentations/typos_perturbation.py,sha256=nfF1Zw2REKZEnnyPVFWD87MP8L5ANbaZXeI2n70Sonw,2790
-helm/benchmark/data_overlap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/data_overlap/data_overlap_spec.py,sha256=aj_l1l0qxUbUMrSWr70-Sb1j_JN-7WYop5BXPG_xj44,1998
-helm/benchmark/data_overlap/export_scenario_text.py,sha256=OiFsU_qME2_I87yDSNPfuAVI5Y9MbPEcEliYK6TaAEE,4527
-helm/benchmark/data_overlap/light_scenario.py,sha256=UFUr8plZD32e2TJTjFZLxTwD-ZRb9eYnHk2U3ZD8P40,1074
-helm/benchmark/efficiency_data/inference_denoised_runtimes.json,sha256=ios_dt-_8wtXvkVAx0iI2zwCxqHvk3XKTx31qHPalsI,4203
-helm/benchmark/efficiency_data/inference_idealized_runtimes.json,sha256=5w7reeZc0yc4cjH8kJGxQQSoe8yaRVX2SSlSrx0QWFQ,12348
-helm/benchmark/efficiency_data/training_efficiency.json,sha256=aH2moiBLStOLVi8Ci2KTK5ZkWlTBLK-B3fRfNZwhoSg,9763
-helm/benchmark/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/metrics/basic_metrics.py,sha256=TnDXzb2sAMlWjZqtqECHRd2wzC3EhOzE84SSulWfUqY,43000
-helm/benchmark/metrics/bbq_metrics.py,sha256=H44mwKXLJ0PXo-sVKgRHCEKjZGnCahDD6GOQMWpnbOQ,6061
-helm/benchmark/metrics/bias_metrics.py,sha256=uVJFQvDSzvPR1ELu0FNYyExjRy2ThaJRCw8beEMDqJs,11309
-helm/benchmark/metrics/bias_word_lists.py,sha256=mx5JjW3mHffXIqo4GcQN-zENUEttBqQnEjPTz3J3J_4,13909
-helm/benchmark/metrics/classification_metrics.py,sha256=pgY8BToHw61lfPTswEW2Da8L4rkWJhthMBYVWnngzuw,5442
-helm/benchmark/metrics/cleva_accuracy_metrics.py,sha256=nSrtxGtIXPFcLrz9mNMYRfJqgQMzoCuf4i3TcQx6Rzw,2580
-helm/benchmark/metrics/cleva_harms_metrics.py,sha256=MrsSAvpOuGGFtVL8RfrG5bNbVTr3LmfhuNfnV4PrAuE,11085
-helm/benchmark/metrics/cleva_metrics_helper.py,sha256=8UwiGhekUmp7DxYWU4rxqX2v3ewkg-O5-jOh49iOGmc,304
-helm/benchmark/metrics/code_metrics.py,sha256=uWdigk0QyEsfVHQzq9KxkOc-LROvcqWXeui42Mr0YF4,5119
-helm/benchmark/metrics/code_metrics_helper.py,sha256=byyuI1lJgbIDPVJzywaBsam9zFMPPyn28g1grsK9xyA,22336
-helm/benchmark/metrics/copyright_metrics.py,sha256=X9j3YsfzWEoGpgPpIvCzm18-JggLAW5QFooifE1KqaM,7729
-helm/benchmark/metrics/disinformation_metrics.py,sha256=0Tpdbx_6Ql-ZCaDKbQDSROMSVG574IxADMVJvGu5TU0,10755
-helm/benchmark/metrics/dry_run_metrics.py,sha256=RFUjmHhxyKWm6fSIZMXFGZWsSlREjvy5KDXYLF5Gnx0,3623
-helm/benchmark/metrics/instruction_following_critique_metrics.py,sha256=-cHgmSKy-Z7ZT4ZFguI4EY6OiEp4MltXBq8zTEXBtaY,9757
-helm/benchmark/metrics/machine_translation_metrics.py,sha256=YOsJWuW7felpZc0c2bS5H9cqnJHokhqVXm1cPzMI5I8,3680
-helm/benchmark/metrics/metric.py,sha256=GEa8xt7SSIaoZMs76bImII1HUxs9mhchIiNFaoG8IIk,18767
-helm/benchmark/metrics/metric_name.py,sha256=hk5WQ6uj_9EjgKKFawPenL2-XOMf-aKvNRkOxlu4nCo,1355
-helm/benchmark/metrics/metric_service.py,sha256=W3BqPoWikPQ8eGTUmcDIqMQK1IM1tymrjL741G2BSjY,989
-helm/benchmark/metrics/numeracy_metrics.py,sha256=panMWD3a1NPerg3Ix7l6NhR7jGOIQOQV9i_KysBeDA8,2818
-helm/benchmark/metrics/paraphrase_generation_metrics.py,sha256=EbdVAJkK9ueA6k4rEdB34jbE_xeh-o164sPiWieLAYo,1885
-helm/benchmark/metrics/ranking_metrics.py,sha256=EZUEbpID-cUO8JI2JzuLzCQ1Brwd5Ty5rzb7XZyGH90,17399
-helm/benchmark/metrics/statistic.py,sha256=FuxNxMtAfiCkOxBS9KHlhEyxe61e0YXt2emvsufgPZQ,3424
-helm/benchmark/metrics/summarization_critique_metrics.py,sha256=Lf7PDuce62HDzyofsyxaOvH0QvzcaS-vJvDWtIs8xKk,4694
-helm/benchmark/metrics/summarization_metrics.py,sha256=8uZ-OGpFIro0U7YHya_c1dv9DFWlkCYhz6lJYd9mVBM,16398
-helm/benchmark/metrics/test_bias_metrics.py,sha256=N22a5UKqSd7TfYBrZS6ciFSOIc1s85pYMwk7m9ufpG0,6274
-helm/benchmark/metrics/test_classification_metrics.py,sha256=usW5ciUYu2ZUUqVjFk4NfZTGNIoBArwia_-8uGOvFpw,5475
-helm/benchmark/metrics/test_metric.py,sha256=S7LGHNCHuhMk582eHylw1tOasUBEf_7F0T4u3tey7b4,757
-helm/benchmark/metrics/test_numeracy_metrics.py,sha256=ls1ZIHDePKpHMoqAbf4HmJ1SIBjLFuLIzGbfg6OiZvM,4162
-helm/benchmark/metrics/test_statistic.py,sha256=AejuYLSeUwEOqpEMRKZFjnxu4HKUraeExU8TPmZEqW4,1229
-helm/benchmark/metrics/toxicity_metrics.py,sha256=6OsuOYX57m6Lvnpe6iYmJAjTGofQ73LBbyCtFCQ2Cbs,4033
-helm/benchmark/metrics/summac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/metrics/summac/model_summac.py,sha256=HNWB5-C0Xnqml4TnXyPZdVegNu4hHJsY2IkzX3rm5Rg,17414
-helm/benchmark/metrics/summac/utils_misc.py,sha256=7_Q1c72cKt8PWtxn8u4R8nB53HK6_JF2nP8bBXYNk-A,1485
-helm/benchmark/metrics/tokens/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py,sha256=WeNP4yiM4TVrD9Kid-uVRmWIVDqETnBsMycZmIBiTZ0,665
-helm/benchmark/metrics/tokens/auto_token_cost_estimator.py,sha256=4BOfjqvYHhkU_ZmDWHMEXJWsrKmCYwXB0anoxQo3bSU,1982
-helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py,sha256=5igmDhWu7H8-qiJnAQqCTQd1WXRewMrpdJdo4qhvSWc,523
-helm/benchmark/metrics/tokens/free_token_cost_estimator.py,sha256=G_6UK6Js_NZ_eqY0ZQnrC9QJVMERGhV1f6v7xq2lM-Y,461
-helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py,sha256=qMfWdOdjv6XVJKp6w4zUbc1cSHyxj9cJpir02dPJH1w,1145
-helm/benchmark/metrics/tokens/openai_token_cost_estimator.py,sha256=fNNiqltzDTZAqn4tJynC-IqLg1a1mp3D8pTX847UkZM,1387
-helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py,sha256=cwNhLb24EyE3y2PqYWQt6IuRakMALaYunu4pU2fEfss,978
-helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py,sha256=6BpOJ7gHMqGGfZFWMW9GfzyNPtRYvtapA0VnsgetDUo,2400
-helm/benchmark/metrics/tokens/token_cost_estimator.py,sha256=fTGUfhHV6yMwpTkCEMTGMxKO8jskqJz4sAtwXT6M_C8,425
-helm/benchmark/presentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/presentation/contamination.py,sha256=5wLwq266sCxT62MdzAXT9V-au6b07HaL44DLj_2qiSk,2788
-helm/benchmark/presentation/create_plots.py,sha256=31keYiMlJcGecwir6dXSRvhlFSlLz8xb2pBNecmdnYs,28657
-helm/benchmark/presentation/run_display.py,sha256=oglbxacY64XRLQUSSrp4leR-nkSW-GAVF3Cv6YlEAi8,10629
-helm/benchmark/presentation/run_entry.py,sha256=J1QgLOP99N7N4bs7nzXWxyU3pOd-a1j8xwL9ag1nP_Y,1158
-helm/benchmark/presentation/schema.py,sha256=XJqjkkHaIeEGdoFwPtN9GamcXFtqaZG0mF4O7DfBV7I,9296
-helm/benchmark/presentation/summarize.py,sha256=BIh1ydZhDbe8e_n6WI6zMrppt9fzrr5tH8pQ69XlJUc,61984
-helm/benchmark/presentation/table.py,sha256=VzVMwsgP3kItAM6FPRUaTphzJ-ZjriiuFbWlO1rJUMU,2879
-helm/benchmark/presentation/test_contamination.py,sha256=8mnzUzxUW9pXUOuLpU4BBBg0V7Mn1d1s4AQgwy6_kl4,459
-helm/benchmark/presentation/test_create_plots.py,sha256=5PPPegMTdBZurxyyUxI4rN13AVsjV3eQrwFqlobJ8UA,1286
-helm/benchmark/presentation/test_run_entry.py,sha256=3DgIo9H9s-3zaUxGnAGItUKAM2GvGFPXh-_lxn6A_A4,671
-helm/benchmark/presentation/test_summarize.py,sha256=pbYUlzYOju0pkFjMsT_nM04oXhPXLfJIpTvPzZow3bg,1367
-helm/benchmark/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py,sha256=Wyt7J5BAvAqC5JTqCW4fh7ex9-itX11P_9rLTocqvtk,4973
-helm/benchmark/scenarios/babi_qa_scenario.py,sha256=S1tPQY2x1I3hQL1JQ6wvUwvKyiSe7SqpRSW6N3_T0mo,5043
-helm/benchmark/scenarios/bbq_scenario.py,sha256=AMUeFLaO2jiB-FnBbe_a5sZ8OXBuG7_3lgld3dn_jtE,9581
-helm/benchmark/scenarios/big_bench_scenario.py,sha256=bSk8Ia4u_6OqMjiyadpYQAWN-8GFWqvd3Ft3JiVGpi8,8081
-helm/benchmark/scenarios/blimp_scenario.py,sha256=o1MDcHT14KFDET4K9otx8pDiIgXrhsD19pvO0mR2ADU,6260
-helm/benchmark/scenarios/bold_scenario.py,sha256=-Gg9C7lY8VFXU-sv5RmNOUZ3NXOJqvZtWsZC2iBEAlo,4110
-helm/benchmark/scenarios/boolq_scenario.py,sha256=rvSp5SwXMCVzBo5BFxfhj1Xv06_ksqKrtTQR7nPiS-o,8013
-helm/benchmark/scenarios/civil_comments_scenario.py,sha256=VO5G-cQ9qctmBN0O76uSewnO_mFslMo5mbR2ZTrjuds,4851
-helm/benchmark/scenarios/cleva_scenario.py,sha256=eNnAVmzAzTwTIrWG-O52084k9OGAKhg3wVIbcDok9xo,58540
-helm/benchmark/scenarios/code_scenario.py,sha256=dYaygIZnoUcDnjvz243ddiO8kOrb0V5VHaVOJgiwocY,12202
-helm/benchmark/scenarios/code_scenario_apps_pinned_file_order.py,sha256=KC-5MQ-d8Nn46aDN4FaPxmd6yk1DtVUmVR-CIZsNCp4,1738
-helm/benchmark/scenarios/code_scenario_helper.py,sha256=TnXAlY-wdAFwIDylFItf0z7HOu93WD6dNThwzZYe330,5904
-helm/benchmark/scenarios/commonsense_scenario.py,sha256=TacucpkudU9BLg6g01JoHjGgh__KSEknyigZpBlx7JM,10727
-helm/benchmark/scenarios/copyright_scenario.py,sha256=qw-ijottZ28DjMqMWSZzL9_FuRDAHf0YmGV27YstmVI,3673
-helm/benchmark/scenarios/covid_dialog_scenario.py,sha256=LqNtfb9XQestluN0Bz7yi-Zo0VRkLjl4F9IOYzhPw3I,3971
-helm/benchmark/scenarios/custom_mcqa_scenario.py,sha256=aZkt1Ums-kIVpLfd4Pfhe7Rf_fZLxjO2ANLL2aN2qVM,1915
-helm/benchmark/scenarios/dialogue_scenarios.py,sha256=MyF8CWkd_L42_gGLqG19YjCvpSpChCHeTj4jMUn_IkI,5630
-helm/benchmark/scenarios/disinformation_scenario.py,sha256=kQi0MVVoSDhx2vOTnUaCIttPXMf8zz7Eld2FD_77tnA,8504
-helm/benchmark/scenarios/dyck_language_scenario.py,sha256=vMxND9wPJenrGlCLhSw5UxOw3TV2Jq8cTmIXGpzEWaA,9318
-helm/benchmark/scenarios/entity_data_imputation_scenario.py,sha256=n2mnkmSeTznEy7S-GVumqpD9bt27yctbuEmtgQrG-Y0,6399
-helm/benchmark/scenarios/entity_matching_scenario.py,sha256=Vl1zWDCCxARMLSGFlQg01rYRgUg2DB_kYGg5c_X899E,7038
-helm/benchmark/scenarios/entity_matching_scenario_fixed_random_state.py,sha256=TklbX7Kx4y-estV-YHUbI5O08q2qCZRrOmX9D3gZS9c,2193
-helm/benchmark/scenarios/grammar.py,sha256=Pb9vEP_0Ki87UdQCj1ym7QWJ24M4DRP6TXB5d3GnhLs,5597
-helm/benchmark/scenarios/grammar_scenario.py,sha256=bl-Cm9caDs077zSu38mzaS9maZ2gM-QazgjOEMFvxYg,1454
-helm/benchmark/scenarios/gsm_scenario.py,sha256=9fV2SEw3ocKNAD-TrDZZTpq4l7mbttQQWbO0YNz4e6k,2613
-helm/benchmark/scenarios/ice_scenario.py,sha256=vvk11cFPGUhg_CcGh3wEfVsGzrvMFgkByN-xcF-OOjI,16473
-helm/benchmark/scenarios/ice_scenario_pinned_file_order.py,sha256=fuirubIdi-rkJMfSd7YoDdBX2q0f5K7GGTN4XVapAUY,1613
-helm/benchmark/scenarios/imdb_scenario.py,sha256=-bfjm_43j0lsfGskz2THNeKABCld90rFTZ5yWt8FTHE,6175
-helm/benchmark/scenarios/imdb_scenario_pinned_file_order.py,sha256=fjW0Gkzg2Y3IAbtYJ3KC7MueWd9U8h0tlcBCqxYmRrM,1621
-helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py,sha256=PuwcuHnx3nCuRYEEzEQ6aRI9i7QAHtfg4DmLAG4M6AM,2205
-helm/benchmark/scenarios/koala_scenario.py,sha256=ihx39W1C9KyOh5MkdMqw1twUwzA8S5QPsBKeQRZcZGM,1365
-helm/benchmark/scenarios/legal_summarization_scenario.py,sha256=DTYYsf-74NFjHvrNa5ZWSgMywq9ss16r5eFZI2-xZJk,8644
-helm/benchmark/scenarios/legal_support_scenario.py,sha256=hu8SlDGrU6oUTQjnrLEagjFNniW96wMahKjnrtuyjzo,3925
-helm/benchmark/scenarios/lex_glue_scenario.py,sha256=vsy8Y03hoCwTJFgBdQf0koxlHDRFm2V9wT04n2K1tlk,10268
-helm/benchmark/scenarios/lextreme_scenario.py,sha256=HEpXLdLLywo6yfXaGF3zRgx1PiGPE2ILnpKYs9Dmsow,20396
-helm/benchmark/scenarios/lsat_qa_scenario.py,sha256=5ocAoqR9G7FkIz_XABi0kusmLQXi_nBsuSyDCZrgJ-U,6073
-helm/benchmark/scenarios/math_scenario.py,sha256=nleTdwfbtjv7HltXAPcpjlRnAv8iiQQjoeKtL8Zr8vk,13843
-helm/benchmark/scenarios/me_q_sum_scenario.py,sha256=WLfED6DiIOUPd4PowydPmrQ4v26OsjG4iHumq7ka9do,4001
-helm/benchmark/scenarios/med_dialog_scenario.py,sha256=f7hTcEpxGfoxJ-WXNrSRrgVfbyuYpB624HbyRuMP-KI,7295
-helm/benchmark/scenarios/med_mcqa_scenario.py,sha256=JdVQuzKM4TRyWT6c1zpsMvNk1RYHnKdy_fRM6W7PCLI,5035
-helm/benchmark/scenarios/med_paragraph_simplification_scenario.py,sha256=e9WRx41XKJI06l3ZGkZe_LKNx-qd4hoWopfDncrr3KE,7591
-helm/benchmark/scenarios/med_qa_scenario.py,sha256=R0iDMzu3U0TnYEOiyjnlOWHGFwhvOJBVwJ5-Hos2eP8,4417
-helm/benchmark/scenarios/mmlu_scenario.py,sha256=wZjoGjaZNERyqSDHgIsv8fIANyUUQZ1ExQAa4cPzmM8,3790
-helm/benchmark/scenarios/msmarco_scenario.py,sha256=5DTWSDuinXVomlUTYfwOijXLifCSeG4be9Tvb7SANQM,34026
-helm/benchmark/scenarios/narrativeqa_scenario.py,sha256=FIzd4CTS6ZTOI_TJ8aPlkXYNfqbTIs13ZUqAt7ie8U4,5608
-helm/benchmark/scenarios/natural_qa_scenario.py,sha256=khKCR8xVhRqJtZoAfGQ9_CV6REkfrYn5ahLpci6rUzk,12540
-helm/benchmark/scenarios/newsqa_scenario.py,sha256=g7bFXWCTvRjofmmUJhx3jmCOvcVwi_P9LSVrNofvMLg,7260
-helm/benchmark/scenarios/numeracy_scenario.py,sha256=cyRvUuNlCAVZRuRFYcerCGIZ6CA4_joXN3NvWkJ6VAM,30828
-helm/benchmark/scenarios/open_assistant_scenario.py,sha256=PH8F8zqYXXakr1xttBtFmENGMGciIpyijK4RCkJol_0,5495
-helm/benchmark/scenarios/opinions_qa_scenario.py,sha256=OyJozmvbowPqN0t1FHSR0QU675qX8359YDmsipagN-M,7358
-helm/benchmark/scenarios/pubmed_qa_scenario.py,sha256=zVL1gb3eVz-LbK2hfdnRR9ItaMSPlAGJorByWLt-4wk,7506
-helm/benchmark/scenarios/quac_scenario.py,sha256=SRAhMp6TAsmTRq6VRONLl3SEayFIe23He_mBhzkZ7qM,6628
-helm/benchmark/scenarios/raft_scenario.py,sha256=H3gST4nbMy7UI0gffyhxXlRTIXfSgznQfNT_dPa2UGo,4537
-helm/benchmark/scenarios/real_toxicity_prompts_scenario.py,sha256=GkgJo_13MWQQQTZbhlknvTR6ZrYr7NEn1WdMZrPs4y4,2400
-helm/benchmark/scenarios/scenario.py,sha256=0WoPHN1nKrxDKNUYj5DATC18zsZ82D6cWXuUcKVf3Uc,7732
-helm/benchmark/scenarios/self_instruct_scenario.py,sha256=jZ2MksT4N_4g_sp5egw7ycrsM-Ya786_RFmiYYdMvG8,2285
-helm/benchmark/scenarios/simple_scenarios.py,sha256=TbdQoeIxQAfcUOrFEGgnFrnSswzIuLPcek5RRV54GTs,1965
-helm/benchmark/scenarios/summarization_scenario.py,sha256=MlNMgsY369DC04nhMUdG2o9Ydi6yze1fGOjC0bK-UwQ,6847
-helm/benchmark/scenarios/synthetic_efficiency_scenario.py,sha256=pzifpsJJbucmTjujNqQnwQa4Y7wpQjkS6QjNXOrgTAQ,3096
-helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py,sha256=1b3e3WpFMNBV3li17-0Ug6QCSKO4qRFaWDF23bYNsvQ,16326
-helm/benchmark/scenarios/synthetic_reasoning_scenario.py,sha256=k8IGK6VABOr6wuha4HynP47peoAkmIViAVhScOtCANo,8345
-helm/benchmark/scenarios/test_grammar.py,sha256=sPlA36sHpThbXgnGlXyOuqHfDPe2epIafmzIeL0nkoU,1364
-helm/benchmark/scenarios/test_scenario.py,sha256=OBcsrF7yCAbd3LNYo2pwFExZmixtdAyWaKRax5ngxyU,1870
-helm/benchmark/scenarios/the_pile_scenario.py,sha256=MIz7vMa8u6aKcCSD63d51trQPpJwCwcU2lQ5w1cgmuY,5021
-helm/benchmark/scenarios/truthful_qa_scenario.py,sha256=iqL-tuqUQZjF9-DKAnI6wV-oLGC_I3aFuofdtJYHt8E,6035
-helm/benchmark/scenarios/twitter_aae_scenario.py,sha256=CRlPxVfkg3HPZV-lUMyCUSFOiAqg5IIPt-dq3qR9LU4,2096
-helm/benchmark/scenarios/verifiability_judgment_scenario.py,sha256=0buyJWLhy6edykqEiLrz0pZz-lA5u1UpqYny5aPuDO4,6011
-helm/benchmark/scenarios/vicuna_scenario.py,sha256=5z82_vz3I5pbWctDzC4kKHmtattkalSWQFURB1APYwg,1659
-helm/benchmark/scenarios/wikifact_scenario.py,sha256=cOVKgDisBdjPcmVMCLhTekdgX3hpDJUT-aKbvRSaMoM,5791
-helm/benchmark/scenarios/wikitext_103_scenario.py,sha256=PAPkmZdC4aIBQ1k29dDvTFBEFaPV1ZR1Ifif4FHoZqs,3087
-helm/benchmark/scenarios/wmt_14_scenario.py,sha256=H_KZVe8s5opDH67ken39r6lD7KWUkHFit6HWmp9uOlE,4374
-helm/benchmark/scenarios/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/scenarios/vision_language/vqa_scenario.py,sha256=w2acgyq0kFhh3iGkyCcHKAqWcEZvNEukCd0Nh5ay8TE,5111
-helm/benchmark/static/benchmarking.css,sha256=7PsUCff8YcoVxyWuALItfB4TZL55T7MbVQpoc9nZznc,2104
-helm/benchmark/static/benchmarking.js,sha256=PDiaC43G82EbOeZKskQkM3ko4tBSYMEHiOAEzzWB4WI,54533
-helm/benchmark/static/config.js,sha256=kIfkgr6gaMdFOAdqB35EvuBohq0DWYSQZbe_pTK09VM,103
-helm/benchmark/static/contamination.yaml,sha256=8kGr_ZfADp7cp7QvJSjF4slrnAcMy6Jh-OdwhrQt4Fc,3158
-helm/benchmark/static/general.js,sha256=qcsntanG5UMWK2vznSVAVFy9zd3BMc8DFfNa7KKezew,3053
-helm/benchmark/static/index.html,sha256=xIJGjMg0qn9eemfdBiNbTI0jzPfBD5x0v8HJF-dMqBc,3561
-helm/benchmark/static/info-icon.png,sha256=P-PW3Ek3NGiRAW5BXOjJRPBfMVqprjAqtQheGWu7zNI,3428
-helm/benchmark/static/json-urls.js,sha256=QnBGT85G8xAClva_mIXYXQ_eoHn-pN-LDJbN65mcTpY,1907
-helm/benchmark/static/plot-captions.js,sha256=bTR8gYx-QqF_RJyKX-L-eQP7hSEtawfJSoADCvgjKag,3011
-helm/benchmark/static/schema.yaml,sha256=emMAIHBBW2V0pRqk0MuGDzfATzEjsIi5wE20cBHqCm4,139389
-helm/benchmark/static/utils.js,sha256=bgN0PT53Dregc-nLmEmAEmg2psufWpS8jTf74WoypHw,7681
-helm/benchmark/static/images/crfm-logo.png,sha256=dDkauL_wJR_Luu7L7pltphS3a9HSLjDkpVLa6C9vcA4,62712
-helm/benchmark/static/images/helm-logo-simple.png,sha256=LtVAC4OgcWgMAob53rTrf7cRDu-O0z85ZOGGj9wR9hw,86133
-helm/benchmark/static/images/helm-logo.png,sha256=GTqbrxJr0oQXbBRq-8v6afY5zB5x0M6PhEbKRIX9qIE,280667
-helm/benchmark/static/images/language-model-helm.png,sha256=mG0-bkdziXeiF0wOGd67y2jnYmVKJYqhD2N5Q8VIF8Q,26563
-helm/benchmark/static/images/scenarios-by-metrics.png,sha256=F7g9mvIYopm-n7sDGg-7I0XCyZvloKsi2wIq1i6da_Q,51331
-helm/benchmark/static/images/taxonomy-scenarios.png,sha256=2MiuCLaxnuHvwsWWJHnZFc-rvoQIi_tNIjDatY7I-Dg,100766
-helm/benchmark/static/images/organizations/ai21.png,sha256=Drkew6Vlwi2_4_S8hjagK2x8smOwLKTNiXIT3rDiurs,10208
-helm/benchmark/static/images/organizations/anthropic.png,sha256=cNi8OdIshIIb8PdodcX8mAj-khaUD0O6nhah-_6nYfs,8017
-helm/benchmark/static/images/organizations/bigscience.png,sha256=fwQAwN1x2Fr_ztD_HZdcOkdFcyxuDjtS3B5-VuRNkuc,19036
-helm/benchmark/static/images/organizations/cohere.png,sha256=7cr4LI8WK9yPryQboyWK_T5baSND-d-tVrlPNflLQMg,8757
-helm/benchmark/static/images/organizations/eleutherai.png,sha256=uUURFF8YWY85mwGoKVEjArO5DUBCy4es5naCXsBzn6c,4526
-helm/benchmark/static/images/organizations/google.png,sha256=BtmXrVQZHr3WH5c8c23ent2FO8aPWeNwO8czl22lDCo,4914
-helm/benchmark/static/images/organizations/meta.png,sha256=VYDp8arkAe2eYRJhAOcIAsZY1qY0hqyOEQDgVMbX9M8,4646
-helm/benchmark/static/images/organizations/microsoft.png,sha256=9e5QFl23yTbnAk8u7lZKaQOf4oPHbr_aiQda5n4MZqE,50850
-helm/benchmark/static/images/organizations/nvidia.png,sha256=hvp1wZMwYxkfrVMvJs73PX71JwY5L8ZvxIH_fL4n6Po,27945
-helm/benchmark/static/images/organizations/openai.png,sha256=P4ZT5ISIlt6Dl0mOp7juSM4Y7dfyRNPqdc0PJuwNoqg,16877
-helm/benchmark/static/images/organizations/together.png,sha256=2Vjbdm0ptogad7eh6DY0UtoVekmdOCRD5rSHBV4DVBM,16610
-helm/benchmark/static/images/organizations/tsinghua-keg.png,sha256=l9SzlZCsLF18BY876wYJcVgiQbgvwte7uoILPDcVwHk,7776
-helm/benchmark/static/images/organizations/yandex.png,sha256=OOCdcKubAP4x7h4VW7z5a-AHPWBiSDTjsIJea6ZiovA,27964
-helm/benchmark/window_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/benchmark/window_services/ai21_window_service.py,sha256=aV8uHeAc5_DmbBnh4tgLEgcC_QxdkPotl8Asyr2K1KA,12826
-helm/benchmark/window_services/anthropic_window_service.py,sha256=BKJ70yXlwKDTqwj2YUn0oq7LCUwSWPW-TZFafXep97M,2200
-helm/benchmark/window_services/bloom_window_service.py,sha256=o7MVedt6khdoj8zikLDuVraEzuoBZk7j4Fzjsas0sD4,1023
-helm/benchmark/window_services/cohere_window_service.py,sha256=dp4KTmbwxdHwxy7Eogfr_p0_F5gayFR86PNhUHaqb1M,7091
-helm/benchmark/window_services/default_window_service.py,sha256=_RkwrS5tDwhVaxFPJCUB_hvjdF_9aJfnEve41dMVnLM,1066
-helm/benchmark/window_services/encoder_decoder_window_service.py,sha256=Dg_rD0RtKIALtpvT4Wi4Am3zgFcIvgxfzdglw5fbbTU,2478
-helm/benchmark/window_services/flan_t5_window_service.py,sha256=39IZX89_tay3bpSGVoWDoekmhW-RUNATQuT-bNyFRTs,915
-helm/benchmark/window_services/gpt2_window_service.py,sha256=d-ys9FHoVI1u1GZOH734JPAOc8W6QZb1N3ZjooKmwz8,990
-helm/benchmark/window_services/gptj_window_service.py,sha256=DTCwWigm1Lfq4-XnfuBqnTj9mymdSAn3HwoUrL5tmII,1180
-helm/benchmark/window_services/gptneox_window_service.py,sha256=C9c45zWlro1SskgQBbrAHBIgfYkfgfKviUPaIgT-8jg,1257
-helm/benchmark/window_services/http_model_window_service.py,sha256=IWIBPA9FRPeHwhZQ32BcskvmuJednKlwsUhMZ32mavU,760
-helm/benchmark/window_services/huggingface_window_service.py,sha256=1765Nz5GinXd5ZOvwmzDBVwOgmIoBDp75gr4MoDmjTw,2250
-helm/benchmark/window_services/ice_window_service.py,sha256=5z52rP-xAF_jckIPoogyoNFW6FQXqaq3SHybCaBuRn0,2005
-helm/benchmark/window_services/lit_gpt_window_service.py,sha256=Pf7vbE7x9_ifVVdm5WKuso73Xraor-bExnpvRAQezFA,670
-helm/benchmark/window_services/llama_window_service.py,sha256=78rtagKCC9056YTaE66nft9cqIvZhZZnMEGS1BplSX8,1331
-helm/benchmark/window_services/local_window_service.py,sha256=9nGiWMvmmqmuky3fe7rAAae9vGQYiD1HlSCtSUXk37c,4547
-helm/benchmark/window_services/luminous_window_service.py,sha256=0w-nyfXXDwnIDBzU3Y84LZPnQyJGiSbvf5Y_MQh-hCg,1791
-helm/benchmark/window_services/megatron_window_service.py,sha256=ZXnhrhnwpBLQ8PHGMO-9XGNQTocrmdG4EhrTfVblEIM,381
-helm/benchmark/window_services/mt_nlg_window_service.py,sha256=7zvEEZhqfdefUUFiDaqbJqrBu3Pt7BBuT_Si-4swV5s,838
-helm/benchmark/window_services/openai_window_service.py,sha256=Oguy_ewlL2Uydq-B1QrqqHWKTenL4mtwtY24RNqhFCM,466
-helm/benchmark/window_services/opt_window_service.py,sha256=ilj1G_pslwYeRZ2fhRMXsg2WjQ4rCfTUwuSmjItH-t4,1050
-helm/benchmark/window_services/palmyra_window_service.py,sha256=cnu0j-jrCpw81xtkBFoWQAs5vWs9Fbh2i8A8fSypnWQ,1142
-helm/benchmark/window_services/remote_window_service.py,sha256=TU0pGOXMRRO5GG1MCYrhPhIQD41gQStttRkyHc-p8Dg,1658
-helm/benchmark/window_services/santacoder_window_service.py,sha256=sfbuAAdhEkQkFON_bHxDI9Ek3jZiwCF6upa41bsjUO4,674
-helm/benchmark/window_services/starcoder_window_service.py,sha256=NHutXRYK2MlzNeQsDdG-M5kJqOwi_PcKOga1LyKKATM,672
-helm/benchmark/window_services/t0pp_window_service.py,sha256=oa1vJRiyFPbkTb8eYnfjZNyKFJSvceipOq0U3Ys5e04,1196
-helm/benchmark/window_services/t511b_window_service.py,sha256=8CvkSfuG_Bg17gAEre_bmM4tUwoi9fAWpPxx6qCJwAE,1005
-helm/benchmark/window_services/test_ai21_window_service.py,sha256=HkpNSaJAClZfaa-bQZ2BrRm1UB_u4sLAGSBlGQqRUD4,8221
-helm/benchmark/window_services/test_anthropic_window_service.py,sha256=Zo3IHsKtE2yf3E29_FgBd_dNbwtgTO6LPArzXYEwI-8,4092
-helm/benchmark/window_services/test_bloom_window_service.py,sha256=fG1oJdfErc0m-r-x7WalQ4mW49R76Wt2F_bUuYDSEb0,4184
-helm/benchmark/window_services/test_cohere_window_service.py,sha256=j36eIUywlVqDyDnDlgKA4HgT8YE-ItmvOqCgaVk3csM,3240
-helm/benchmark/window_services/test_cohere_window_service_utils.py,sha256=sf25f9MeXzoqsbDzZ7d7le13hm8RkDe54nhLtKF2pqo,158150
-helm/benchmark/window_services/test_flan_t5_window_service.py,sha256=xv_EXbiRklveJPQtThYCSYF7qBYwjL7K4wH3Xu5z2Fg,591
-helm/benchmark/window_services/test_gpt2_window_service.py,sha256=3k25pLa_z__g4yoQL40DEXj-T4dGtrgif13N2NXs59U,2568
-helm/benchmark/window_services/test_gpt4_window_service.py,sha256=IR8JGLViwiFDmHOLgBfnPGeAJlV6IyRKt1BHOQ4P4X0,948
-helm/benchmark/window_services/test_gptj_window_service.py,sha256=sxsTpozKv9N-wZXtGl1prkQr9Md_q-tnCjO9zt226Co,2267
-helm/benchmark/window_services/test_gptneox_window_service.py,sha256=VwawHQnvrUKUr9Tl0eKjNLtZLEmpJPGxAUsAYMY1hlg,4103
-helm/benchmark/window_services/test_ice_window_service.py,sha256=hSJ52MQ2Edh6gTvgzCHFMvlZuidRhZF_t2E-RR4H_6A,23475
-helm/benchmark/window_services/test_mt_nlg_window_service.py,sha256=2PtD1orlVYYNa_JOV0_3bQPct3zvUx0YWzTnwwKZ74I,2212
-helm/benchmark/window_services/test_openai_window_service.py,sha256=q4ou-AK7-NvUkwAQP4ZuldrAgoGeDnfqM8Syj-7KZfA,2208
-helm/benchmark/window_services/test_opt_window_service.py,sha256=Fn_-q-yfh8wHhIdJ4ysTDmRkbz5S8I806NWLwRlTcW4,4108
-helm/benchmark/window_services/test_palmyra_window_service.py,sha256=WYSpFyr8Ll0-umMnqvUbIz69Vs4urqH4regmeC4sed0,4108
-helm/benchmark/window_services/test_t0pp_window_service.py,sha256=O99CgeJWEqfh5t6__jr_RrcYUvm2JplDbD3MpN9Ay4I,3894
-helm/benchmark/window_services/test_t511b_window_service.py,sha256=YtwycL2-oPLkWkZduSkm7AWGWzXJQC9A3sLjo7dy1-w,3897
-helm/benchmark/window_services/test_ul2_window_service.py,sha256=otiqcOPPFDgPzB2avNOvdmJwVUZH443J2KiQQ787xcI,3957
-helm/benchmark/window_services/test_utils.py,sha256=TMmPTkJ4vlFKv4vzSO35B6-0gnXbQ0nvNZJR6CZN6qo,3227
-helm/benchmark/window_services/test_yalm_window_service.py,sha256=NDvZyDkgO-eYjt1oEieAJiNFFgt9qqXAa38CqWaWq4c,4197
-helm/benchmark/window_services/tokenizer_service.py,sha256=dPsEhA1I4SUvsR0UBdzJ2wsFahw8GuF6qhE8CxiPZRY,1051
-helm/benchmark/window_services/ul2_window_service.py,sha256=R_VEzOb59zQE9mmbTLunQeIvLAtK3-97h-B2_oc0Uxs,1021
-helm/benchmark/window_services/wider_ai21_window_service.py,sha256=o52sBY0OgHEiPrMv5rve8FzE4uXy_QuDFdjTgqQpZM8,727
-helm/benchmark/window_services/wider_openai_window_service.py,sha256=F5Uox-MROlmvwLN5tPa40oe1ozU12jIicBhvV36xhsc,1471
-helm/benchmark/window_services/window_service.py,sha256=YmMyFRBkQ1sH2bGPEuUBC7x4nt6n8wKRVWyENlr6B28,3361
-helm/benchmark/window_services/window_service_factory.py,sha256=Wrv7C_lICND8nwKAq1v6JWNkhV59tvpyrHocBUdBUko,14794
-helm/benchmark/window_services/yalm_window_service.py,sha256=tPOAiUsz3Z6aAU-nBq81lLDtYttZopRGJEcLNfCVBak,1813
-helm/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/common/authentication.py,sha256=RlMx29_TSrfU7ujE7dJkxmFub5EqLj2NswV5lAVFFDk,179
-helm/common/cache.py,sha256=sDMK5z_R_FvRXpIunyv5L7imqpqbId2AuUO2ncKZMUE,12973
-helm/common/codec.py,sha256=gTh6AwIQ0Bbul_QSnIO7eItwMZmYtnkIrG1jkc4GOL4,7100
-helm/common/critique_request.py,sha256=Exu8Ans05zCU5d5-AglEbG40mBwKYED2Z3WqY_XjXBY,2772
-helm/common/general.py,sha256=zGdo660mOkGVFDv9sY_O137sC6v3A-yiGK97_xsgm8Q,11403
-helm/common/gpu_utils.py,sha256=pmLq6ipYNLEm28VxxSNeZuVt-gAw-WnYmBvxP1P1p6M,480
-helm/common/hierarchical_logger.py,sha256=EnKLnfbQftca08EJfjGEQb4tcnCKbx-JtwLnoCnhMQs,2908
-helm/common/images_utils.py,sha256=Jp6nlqYfrBvtwuWdWThQvuSHWWw1qM2WQH7KJfxQ0VI,1514
-helm/common/media_object.py,sha256=nAl4phYCAl9TL1q61TlW4zwInDcdgwzYr2X4JZfFJSU,4709
-helm/common/object_spec.py,sha256=tyPfEisDRq6dq2mfuRoSnFfTqyxROa_3Bos8CNjVwJk,4290
-helm/common/optional_dependencies.py,sha256=Qam3QCHff8tuXbS-fCw-MVe-pK18gSvHw-uQoXXxT7M,616
-helm/common/perspective_api_request.py,sha256=WAVwtajNVmi5XJNsPcorGEAVrqkpPSk-Kd3b0hJghbA,2427
-helm/common/request.py,sha256=Qf2MdWBp-a0oVriKk1dVy9AiP9trcWyO4y_dxjZTmEY,7268
-helm/common/test_cache.py,sha256=XqboYHQAkFWIHPsuIjuageRSLeN7QoATKF7wwxggPqE,7054
-helm/common/test_codec.py,sha256=igL--k-2DwAy0eoMr8D9Xs8MOjBoT0LutbMPzDlTNkM,5885
-helm/common/test_general.py,sha256=zOxSwWNgWnWHsXKcG4NZ50GkWicn4uZ4jPVypSwFaQE,1672
-helm/common/test_media_object.py,sha256=AAm9DD7MC-ZvTwiqXA-e52U6L4S1noxItW8f7ARi6DY,1650
-helm/common/tokenization_request.py,sha256=DS-s6enWjGk9qymcEYFSTTNlxNnB6VoJclW5B6Y6Hzo,3407
-helm/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/accounts.py,sha256=AZ0hCCD3ygSXtgZtnFJFJE1jOSRh5gMtQSM8TeJzqn0,13533
-helm/proxy/cli.py,sha256=l8F7UYqrIOoBD9ZCIxJFA4fhxlzhae0-2Nn8A7FMkzk,8244
-helm/proxy/example_queries.py,sha256=p1wH-tp1pRUslkAwaJYrrG5aDfmFWK3KYn4M6WQfPqQ,4120
-helm/proxy/models.py,sha256=d8xCbT-2_AHpxYSJlszAQ35Jh_bcMs8nZIyjBNQb7Gk,31257
-helm/proxy/query.py,sha256=eftbiUICMh8QIHVs-7cLtv_rDXKeKdRPmwjLMu0TDxQ,645
-helm/proxy/retry.py,sha256=iu0d2Pzg1YOdY6VvmujC5qM4HFPJOSEtOJ82Scki32c,3243
-helm/proxy/server.py,sha256=E6eGvNUQ3JP9ekSb0xPUrFS8pDnssCaO174VL96MQrI,7786
-helm/proxy/test_models.py,sha256=hWeDcBw1GkPvyJUd-ABxRVe1FhSUfz8bzyrKYdsqmyY,726
-helm/proxy/test_retry.py,sha256=8h398auzjW9VnlTJWllxR-bdpub-XFp8EN8LWDEnEHM,1049
-helm/proxy/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/clients/ai21_client.py,sha256=0FmySyVSZsG3W8wYdSYoNTGHeTPtXGX3O2Bu6kOtN5E,5962
-helm/proxy/clients/ai21_utils.py,sha256=mlg3h615kyckccGZv9rqsP4Y60O3XpwyE-UURRMrxII,471
-helm/proxy/clients/aleph_alpha_client.py,sha256=Z87SCS8zahVAkRwkBiJY4bbZSAr1OshiRN53CgYSme4,4214
-helm/proxy/clients/anthropic_client.py,sha256=AdhezmHhUMVCXnPi4SjfUBCjS3pmJol2MYRpBAl94J0,23898
-helm/proxy/clients/auto_client.py,sha256=RSuVenztW1c_0ggbb5t5uN2CUXCO7ZDQ-LDpg-yvHEY,21021
-helm/proxy/clients/client.py,sha256=Ke5Lb1tiWnqtMNffncj9pqf9aEQOrIuqq2n6bvoDiKc,7754
-helm/proxy/clients/cohere_client.py,sha256=5gTz0iOvxPaFZAryxATBLGtmnnaHCnNYMadMR-ET0d0,7867
-helm/proxy/clients/cohere_utils.py,sha256=aYmj60m0e9RF9BIdxp1vmA-uZv17TEALw0dbgTUSpCc,504
-helm/proxy/clients/google_client.py,sha256=GkKG1qJsTESEIjIUeoJqKSxBNFBYMdB3Q8nzPuC4Kyw,3003
-helm/proxy/clients/goose_ai_client.py,sha256=iBnPV-J72rLiy8mK9AipwKGNcGWuPWP9WbU9KjK5_C8,3914
-helm/proxy/clients/http_model_client.py,sha256=-0ODVlBMJkwv0AST4ayDB4rpmLCljIGaYWIiKoVEGow,2931
-helm/proxy/clients/huggingface_client.py,sha256=omg2KVA0yja0WIpRcRamARnFASK9oHYRR7CNjBewmo0,11192
-helm/proxy/clients/lit_gpt_client.py,sha256=ntSm_HIj4Ygq-5iFLw0SIN1pJ82FW5wS7SYYqCQ-4Xo,6171
-helm/proxy/clients/lit_gpt_generate.py,sha256=8DdBE9ReQ00NbV3KMFYc--PlO9X-HMOR0Rhm5CADWEA,3103
-helm/proxy/clients/megatron_client.py,sha256=zOPxdkZ4KMKeKy3XuCDAvd6kWhg2oTx-kpWKE_zgLSM,3870
-helm/proxy/clients/microsoft_client.py,sha256=vN96_BGOvuKgK15pMCF3ejNeygNiNxWUlSOqASfPne0,7943
-helm/proxy/clients/openai_client.py,sha256=gMWT6UShPHOr7Vcq2lZKIdriiazpHMODT2TVVVHDGh0,9847
-helm/proxy/clients/palmyra_client.py,sha256=CfTebjpC3134cQmEAkJH6SBsJH0HGkdlsQl16tbbAjA,6412
-helm/proxy/clients/perspective_api_client.py,sha256=wwfD62GdxKsULZNXHuzx-_JUde96MFIBwZw5QGrVzc0,5880
-helm/proxy/clients/remote_model_registry.py,sha256=z8dWxbPmjmIyBnw--NVzqeIXUj5d_w5qolJD__r3mPw,962
-helm/proxy/clients/simple_client.py,sha256=-ru2ZfdUHhi1ZJpHKt-9YBJoKefNm1uPtltKje_AJFs,2221
-helm/proxy/clients/test_anthropic_client.py,sha256=Kb3PxAZBUeM0escyBh68VfqF6IbGe7SAFqFdC-6TA3Q,2887
-helm/proxy/clients/test_auto_client.py,sha256=EP2ZINxsUsTvgRU-TxDHGnmJJI--MuHpD-sW-v-YvUk,3481
-helm/proxy/clients/test_client.py,sha256=bvkFob_Yoy8bALrVeQ0h757g9RU687JYI0g3AISPFQ8,1268
-helm/proxy/clients/test_huggingface_client.py,sha256=-WRW5z6sgeFsdvd3AdV4PeLBUL_TB0b1hvIqG0Y8xCc,3744
-helm/proxy/clients/test_together_client.py,sha256=nTeL7kofrZ-1GUKXRHLDzn1PdnpcGGA2RgdLiOWGFhA,3381
-helm/proxy/clients/together_client.py,sha256=UurRkVCNUs3lntOAtSoUqhDOugwcGz_DPOGVuQxeDV0,15484
-helm/proxy/clients/toxicity_classifier_client.py,sha256=AI_FizxMurubTIyeceRdkixSnhWQbcD-oEEONj5ve7o,464
-helm/proxy/clients/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/clients/vision_language/idefics_client.py,sha256=8XBVG1fVXu_RE6KdYgWgWgzSV-tZS-D-a7KoSRCX2Jo,7133
-helm/proxy/critique/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/critique/critique_client.py,sha256=ATZuXw77lejwtpgLg3Soy3VDyv8D8xetl0o4guxDM2M,1764
-helm/proxy/critique/mechanical_turk_critique_client.py,sha256=OcppmFOMweBSfVTiLIICIwjvPpHHTkdu9fFUTaubitQ,574
-helm/proxy/critique/mechanical_turk_critique_exporter.py,sha256=taULrc_cIP0O9c5UpGz3l9DmWQadTVzN_v-qzTgMoyo,8470
-helm/proxy/critique/mechanical_turk_critique_importer.py,sha256=fhMf4O9O712GEQD2lLv7HQDtN2BARhOCrnDmlMYbwmw,5535
-helm/proxy/critique/mechanical_turk_utils.py,sha256=mKpUv4zz3s5ptzDY7UrwuI7Cr5HmNgSjPC10BnN9AL4,1766
-helm/proxy/critique/model_critique_client.py,sha256=Dy7n8RY7LGd_fyHiNuNublr_pE6u5eeir1YaJrhdkvQ,10810
-helm/proxy/critique/scale_critique_client.py,sha256=B4povtceyfal95eE3N7em9cC_B5Vy4jMrHXcsXc_5m4,15889
-helm/proxy/critique/surge_ai_critique_client.py,sha256=HnzgAoF4Du9Me0GS_lbNaozZslS4a2OZx735gh-coo0,8357
-helm/proxy/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/services/remote_service.py,sha256=YudZE8RWX4cB0YqYCC8yUPLrT_nfZmaSpD1tubhkT-Q,7395
-helm/proxy/services/server_service.py,sha256=D-no8WSFJO_uAZu94ikdrPgltbnXdjJS0F1mFc3YWDo,7583
-helm/proxy/services/service.py,sha256=s30R8o1SygZp81vRQsFn-9sDp5aoe8QP3CJv-TWFEHw,4730
-helm/proxy/services/test_remote_service.py,sha256=u9Dt8LLY5H67rPLK_WNeQMRxqIxsz1axz3lRHt-9yNw,6425
-helm/proxy/services/test_service.py,sha256=9SgIf9iW7zhQ55KqnDBuZQH2ECnBti8eK2Zps-JNlzw,8751
-helm/proxy/token_counters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/token_counters/ai21_token_counter.py,sha256=Y0LmjyLarUIlshQIhcsvD-nUJQ1KmEAW-jJuinYAJrQ,962
-helm/proxy/token_counters/auto_token_counter.py,sha256=r1lR--_-mc8p8U4hnvL7oqw-fYcIFr-_t_LckOaBTIA,1847
-helm/proxy/token_counters/cohere_token_counter.py,sha256=DHzZ0V3CnUvWvTdgCBjC-SEBbYxu_A7kfXsRdaFr0ro,488
-helm/proxy/token_counters/free_token_counter.py,sha256=DVdKgaQDxd42A8D9elcSunmtxAAtzUr1sEPRLJQ4ZSk,399
-helm/proxy/token_counters/gooseai_token_counter.py,sha256=cY5-3MJcTf0Wzpk5c3-8C8rXkEmdr4yBPv60SuBFtPI,1071
-helm/proxy/token_counters/openai_token_counter.py,sha256=gPo_VrkEH07xmprzdfIhmJ_1iPAxX2hBCFSDYCXBn-Y,1060
-helm/proxy/token_counters/test_ai21_token_counter.py,sha256=42J1fCi20kQUwAD18bIa6h9TaP7KZnlgF-mLbvKURro,5508
-helm/proxy/token_counters/test_openai_token_counter.py,sha256=PJW77e_n2hRmkdU_nDWC0TZ-kL8RCpW3UsE5idwI4Lo,5062
-helm/proxy/token_counters/token_counter.py,sha256=x8KyTR82EedgCQUuneQiVq9AiU1B3_CHPmKPNumClHc,429
-helm/proxy/tokenizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/tokenizers/ai21_tokenizer.py,sha256=snBAumRIGwhTFm3oeyVexpla7g6paeTIIIMr51f-S44,2238
-helm/proxy/tokenizers/aleph_alpha_tokenizer.py,sha256=pEQKdX0hTbmPpq0RLnxQoJ_fH2zaRFgtqj85jTLXHhw,3683
-helm/proxy/tokenizers/anthropic_tokenizer.py,sha256=eaUumCgxmF4N0kloVvYjSWWXVHeYMST-h6W_XESyGlM,1767
-helm/proxy/tokenizers/caching_tokenizer.py,sha256=71W2T9wHEgVLI6cBfJ_Bo9g3GwUa-ujCd3PrQdMkoTE,7812
-helm/proxy/tokenizers/cohere_tokenizer.py,sha256=9mRLdwJhy_JgUW-y6-CFc_iez2raky3S7UqPJ0dcCbE,3745
-helm/proxy/tokenizers/http_model_tokenizer.py,sha256=nfIANZt_0UCh6SM7VZI72wmBov80PkE38v5kgeVPD70,3062
-helm/proxy/tokenizers/huggingface_tokenizer.py,sha256=aGw-1UAhROZV_5OhtvzfXwd8S_8HSBrVeeOAdbUvaCE,7448
-helm/proxy/tokenizers/ice_tokenizer.py,sha256=4ZTIRpmt2cqwcxnmrDpCRhiJ0BI3ELE-GHoBuHWgrDA,1200
-helm/proxy/tokenizers/lit_gpt_tokenizer.py,sha256=LMrpaje64UmnDKoYjPG_RQeXVA4xQUwW5t48IJIeLaQ,1660
-helm/proxy/tokenizers/simple_tokenizer.py,sha256=GjQz0D1IQ3reXO5e6fkYkn7a9b_vmmIKch2wT13qljE,1134
-helm/proxy/tokenizers/test_huggingface_tokenizer.py,sha256=aDcDuqg5yvkf33TdzvOpQTqkRJwU0prutddauZTCIE4,2521
-helm/proxy/tokenizers/test_ice_tokenizer.py,sha256=-xi_f8TBSkAYr5CcA56HDq7rZ9HAGd99J7twNfkLzFU,2619
-helm/proxy/tokenizers/test_yalm_tokenizer.py,sha256=qWpKnUuAlePd6t-UJB_mAiBwtAacnC8caKXLJ_GdTkk,2477
-helm/proxy/tokenizers/tiktoken_tokenizer.py,sha256=FU2g_FF0pVoyspYhHcz3SyCBGNbsTby-nWVrj0Cq4_c,1265
-helm/proxy/tokenizers/tokenizer.py,sha256=1zi9fhgrs2znINKDnoEhW-6UatsVj30hRjQbTDbOdZg,1583
-helm/proxy/tokenizers/yalm_tokenizer.py,sha256=uN7ZLiqG2uCmL1iWy6Y_Je6XuATyvqupWPjJDLnjdNk,1411
-helm/proxy/tokenizers/yalm_tokenizer_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-helm/proxy/tokenizers/yalm_tokenizer_data/test_yalm_tokenizer.py,sha256=W9p5QNn1GSm-y85yVEQe_82zn5CVK_vR6jvhk7JTs_k,869
-helm/proxy/tokenizers/yalm_tokenizer_data/yalm_tokenizer.py,sha256=wXP-LmFLUjRfCExkNHbwVUINE8qFPwvv9h7Sf_pVNbw,6395
-crfm_helm-0.3.0.dist-info/LICENSE,sha256=bJiay7Nn5SHQ2n_4ZIT3AE0W1RGq4O7pxOApgBsaT64,11349
-crfm_helm-0.3.0.dist-info/METADATA,sha256=J42CcYhRw9LJbHnC-pR4_V-KbGcEkZz6Fjl_Awtmvbw,13308
-crfm_helm-0.3.0.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-crfm_helm-0.3.0.dist-info/entry_points.txt,sha256=AvH9soAH3uey9xffisWewd0yrmPWGASC036jHd1SFyg,300
-crfm_helm-0.3.0.dist-info/top_level.txt,sha256=s9yl-XmuTId6n_W_xRjCS99MHTwPXOlkKxmTr8xZUNY,5
-crfm_helm-0.3.0.dist-info/RECORD,,

helm/benchmark/vlm_run_specs.py

@@ -1,71 +0,0 @@
-from typing import List, Optional
-
-from .adaptation.adapter_spec import AdapterSpec
-from .adaptation.adapters.adapter_factory import ADAPT_GENERATION_MULTIMODAL
-from .metrics.metric import MetricSpec
-from .run_specs import run_spec_function, get_exact_match_metric_specs
-from .runner import RunSpec
-from .scenarios.scenario import ScenarioSpec
-
-
-############################################################
-# Prototypical adapter specs for VLM evaluation
-
-
-def get_vlm_generation_adapter_spec(
-    instructions: str = "",
-    input_prefix: str = "",
-    input_suffix: str = "",
-    output_prefix: str = "",
-    output_suffix: str = "",
-    max_train_instances: int = 0,
-    max_tokens: int = 100,
-    stop_sequences: Optional[List[str]] = None,
-) -> AdapterSpec:
-    return AdapterSpec(
-        method=ADAPT_GENERATION_MULTIMODAL,
-        global_prefix="",
-        instructions=instructions,
-        input_prefix=input_prefix,
-        input_suffix=input_suffix,
-        output_prefix=output_prefix,
-        output_suffix=output_suffix,
-        instance_prefix="\n",
-        max_train_instances=max_train_instances,
-        num_outputs=1,
-        max_tokens=max_tokens,
-        stop_sequences=stop_sequences if stop_sequences is not None else [],
-        random=None,
-    )
-
-
-############################################################
-# VHELM run specs
-
-
-@run_spec_function("vqa")
-def get_vqa_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.vision_language.vqa_scenario.VQAScenario", args={}
-    )
-
-    # TODO: finalize the adapter spec parameters once we add more models
-    adapter_spec: AdapterSpec = get_vlm_generation_adapter_spec(
-        input_prefix="User: ",
-        input_suffix="<end_of_utterance>",
-        output_prefix="\nAssistant: ",
-        output_suffix="<end_of_utterance>",
-        max_train_instances=3,
-        stop_sequences=["<end_of_utterance>"],
-    )
-
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
-
-    run_spec_name: str = "vqa"
-    return RunSpec(
-        name=run_spec_name,
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=metric_specs,
-        groups=[run_spec_name],
-    )

helm/benchmark/window_services/anthropic_window_service.py

@@ -1,68 +0,0 @@
-from .gpt2_window_service import GPT2WindowService
-from .tokenizer_service import TokenizerService
-
-
-class LegacyAnthropicWindowService(GPT2WindowService):
-    """
-    This is the window service for the legacy Anthropic client.
-    Please consider using the new Anthropic client with the AnthropicWindowService.
-    Anthropic has their own tokenizer based on the GPT-2 tokenizer.
-    This tokenizer is not publicly available, so approximate with the GPT-2 tokenizer.
-    """
-
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of the Anthropic model."""
-        return 8192
-
-    @property
-    def max_request_length(self) -> int:
-        """
-        Return the max request length of the Anthropic model.
-        Anthropic does not include the start of sequence token.
-        """
-        return self.max_sequence_length
-
-
-class AnthropicWindowService(GPT2WindowService):
-    """
-    Anthropic has their own tokenizer.
-    """
-
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        Return the max sequence length of the Anthropic model.
-        While the limits seems to be 8192, we limit to 8000
-        according to Anthropic's recommendations.
-        See: https://console.anthropic.com/docs/prompt-design
-        """
-        return 8000
-
-    @property
-    def max_sequence_and_generated_tokens_length(self) -> int:
-        """
-        Return the max prompt length + max token length.
-        Anthropic is one of the rare models that has a limit on this.
-        The official limit seems to be 9192,but using scripts/compute_request_limits.py
-        we found that the limit is actually 9016.
-        """
-        return 9016
-
-    @property
-    def max_request_length(self) -> int:
-        """
-        Return the max request length of the Anthropic model.
-        Anthropic does not include the start of sequence token.
-        """
-        return self.max_sequence_length
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "anthropic/claude"

helm/benchmark/window_services/bloom_window_service.py

@@ -1,35 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class BloomWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        The model was trained with a sequence length of 2,048.
-        Source: https://huggingface.co/bigscience/bloom
-        """
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length."""
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        return self.end_of_text_token
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "bigscience/bloom"

helm/benchmark/window_services/flan_t5_window_service.py

@@ -1,29 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class FlanT5WindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # We subtract 1 to account for <extra_id_0> that gets appended to prompts.
-        return 512 - 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "google/flan-t5-xxl"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""