crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

crfm_helm-0.5.0.dist-info/RECORD

@@ -0,0 +1,642 @@
+helm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/annotation_executor.py,sha256=ZJCc5xT8E0E6gux8dq3HPS4YzQs2YPCNl4gx44ghuJM,4862
+helm/benchmark/config_registry.py,sha256=Cd25a8FHriUzAgvGGU5sBAPyhisdSIjdUJR4YbYs6T4,1603
+helm/benchmark/data_preprocessor.py,sha256=aNdM-o2t4qkLIQHiQeWUFg03DjjJ8HTBIphYCK8pXVo,2173
+helm/benchmark/executor.py,sha256=simd7SdJ7TciUpoq3D0uz_XUSCZj5KIWCIP57FYm4js,4906
+helm/benchmark/huggingface_registration.py,sha256=RzfOaLAnzAcoTphan1JNo836lNyxMSH67oQlolhNLS0,4154
+helm/benchmark/model_deployment_registry.py,sha256=BjL0ghHgO7_Z5jZZ7kuSOj9saegI3BivaL-b699C0rc,9527
+helm/benchmark/model_metadata_registry.py,sha256=q8-tERzoY6i6qFrKs9IM4R3BRjO31t0gmdT0lpCmdRE,8009
+helm/benchmark/multi_gpu_runner.py,sha256=WmTKpVfcKXyiiPzrmxpbvQoZy0Ua8IyPgxB8r_3jrRw,4773
+helm/benchmark/run.py,sha256=tF_aWy5GtfwBOT1ZRKWrcI74VpFWGzlR00EKiGG7zyI,12572
+helm/benchmark/run_expander.py,sha256=NiwP10enxRtBj3yJys1AYCvVUBD9CG4mrf_L6QVBRsk,47965
+helm/benchmark/run_spec.py,sha256=GiIU8iGO2FGYFDWIxt51CeNPsW7rM7BzDqH1KgEL1cg,3217
+helm/benchmark/run_spec_factory.py,sha256=KQIUmoNa6iHc-xp8-lDMG1PhfgE2_eTFejyyZ1jXFj0,6874
+helm/benchmark/runner.py,sha256=zlHDJ2Ys5-HxtXcwpkXcrdfXy_i886fBcq1iNeLyC3Q,14669
+helm/benchmark/runner_config_registry.py,sha256=2gW5wBLkHdYb2WNbZulto06hTcto2ROvjy8HULw3jNM,515
+helm/benchmark/server.py,sha256=ysd5MT1TDu65NH-OzIGf9wmZlr8FHNRwoy2ybjSc5Yk,6140
+helm/benchmark/slurm_jobs.py,sha256=eNCAoaWDfT0Wk32ZJRIGo-x8kgjhDPnPB4Xrvw_eLB0,3225
+helm/benchmark/slurm_runner.py,sha256=Tozimrjr2R6mlKHcmrGgxTy9ga-ArIW6AoAWtxqzw-M,16567
+helm/benchmark/test_data_preprocessor.py,sha256=_esdtkqyU_8Yp5ZOO7n1b-Y4Qc28wpD5drG-4Y4UhIM,2219
+helm/benchmark/test_model_deployment_definition.py,sha256=wGN95ku-mROh-yiHH3bL8GC7OWkBOa0YJCS2RIGRv8k,4468
+helm/benchmark/test_run_expander.py,sha256=gLeHkNt_nLgbwEJiYxhwda-eKA3sJAxkYolCvgRN5TY,1163
+helm/benchmark/tokenizer_config_registry.py,sha256=ZOImg38ta0FXZYAWna6q7A5xrG2mU7Ofr-8j4EqGlUY,1585
+helm/benchmark/adaptation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/adaptation/adapter_spec.py,sha256=tZ40ovgNkRsxDOHan4lcD8ukutA1QPsoZUF5XOHq-VA,4382
+helm/benchmark/adaptation/common_adapter_specs.py,sha256=-ILsVxWjpEE6an1ncrRRrLkdP5ky_-2GN1TxSxJo38M,10449
+helm/benchmark/adaptation/prompt.py,sha256=n0Ka3RGSWMr3CBnJrPNPy626x9TJE3k677wKbG8hO9A,2133
+helm/benchmark/adaptation/request_state.py,sha256=WAPyubn35on-Ry7xKpXsVz3wYBMCMc_LidDOdcKxatI,3053
+helm/benchmark/adaptation/scenario_state.py,sha256=mWEhgzk18SVoMEuj2pSnc_r9JrGAHLdOlteHJKUMA5k,1961
+helm/benchmark/adaptation/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/adaptation/adapters/adapter.py,sha256=NbsvNITD_xByHxwknkHS_vWrzvO1peA9T1rjWbz_cy0,1239
+helm/benchmark/adaptation/adapters/adapter_factory.py,sha256=fJgOdJnaVNoBu-aP7GCoXlk8tOY1wyFgDMJVvhiblDQ,2837
+helm/benchmark/adaptation/adapters/binary_ranking_adapter.py,sha256=zF5KcGip5CAMfg5mzisYQ2jlIuChbraujN-UaxCV8E8,5782
+helm/benchmark/adaptation/adapters/generation_adapter.py,sha256=F7Aou6r9CZ1xEuAXxfxgQa6hrbrnuBtE7tgmCgOa7N0,2146
+helm/benchmark/adaptation/adapters/in_context_learning_adapter.py,sha256=BbcBEJjY8Cp58me9sUktd2p3dEVFL8ZJ7RFfus3hSYE,14997
+helm/benchmark/adaptation/adapters/language_modeling_adapter.py,sha256=LhZHmciP8lAfu7T0p634GOPTHrJR7qRCRRIxPgVlW9E,14873
+helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py,sha256=VJ66MfIGQWJg0VXCV0MJEMwF9Jx1DeJ7RxsgYlOTx_4,1889
+helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py,sha256=k8wSxv9pK8wtbQNBzWYPkGEUKJb8tcVi41Y1M3fPQT4,3985
+helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py,sha256=aMe-y4iiyEnM7_bqIoddeZBsVLoDxMmjKY2eZKB6Y2Q,2156
+helm/benchmark/adaptation/adapters/test_adapter.py,sha256=0-JrYnogZu4kENQG1eQMXHWnuSurCLRbkLpDuSnfRqs,745
+helm/benchmark/adaptation/adapters/test_generation_adapter.py,sha256=TM6WJpWShsu6KuDzlofYHd9DNPj86Hjudubp_sqrhFI,12734
+helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py,sha256=f_bggObKUxiV5XyYHHNXsM42HzM0CDzvR4uiIoXTE5o,7997
+helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py,sha256=v6LLmVTopXNfzo9Qzq16EmmPPivFGGs9LuaPDJAX4vY,9506
+helm/benchmark/adaptation/adapters/multimodal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py,sha256=o7CGClyVWYOuJ4G56-whq5fTvCr7QIn51Mo6DTdvwg0,1881
+helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py,sha256=9_9PgNIiJDoTm1NIhCKWytCYe3hHzHJUhaAdd8hM0PQ,6300
+helm/benchmark/adaptation/adapters/multimodal/multimodal_prompt.py,sha256=5jdVVfWf0lBErwzDamIHozZSbf1mEkpmACX8mPlHJc0,1907
+helm/benchmark/adaptation/adapters/multimodal/multiple_choice_joint_multimodal_adapter.py,sha256=PWI_WrfDolipj7Zs43YxFQk36jBgU76PU-kL8R9gRno,4759
+helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py,sha256=VjSqWiZEcW6K2jrokGUmky7syEOqJ6cbHImR7YZgwzU,10151
+helm/benchmark/adaptation/adapters/multimodal/test_multimodal_prompt.py,sha256=2OZhhOtEZ7RvItHZD4AGsefXvVUwEQZL2T-ESHPsvdI,3482
+helm/benchmark/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/annotation/annotator.py,sha256=2UIXY71S5dRaZBLb1v4lcv8-O6pyJ9zTeSJl78AEWGI,1538
+helm/benchmark/annotation/annotator_factory.py,sha256=z5AGBylIuy-_IfgikX66VyGvRz4SxtnOcJsyESH8990,2699
+helm/benchmark/annotation/test_annotator_factory.py,sha256=ifv5hxSbFe113AHeXLqTPkVJ-C2PW_gb9L3a0SHNi-M,986
+helm/benchmark/annotation/test_dummy_annotator.py,sha256=LfY1ErJDUJ7rD8JUy92RUDD1b91jUs4Nk8Gvope-Z98,1644
+helm/benchmark/annotation/image2structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/annotation/image2structure/image_compiler_annotator.py,sha256=eJFm3iyBe_eEN5Yt0G2IpeA1xdKxRmyR4krsNd6eXoE,3524
+helm/benchmark/annotation/image2structure/latex_compiler_annotator.py,sha256=yRifoqhGq_mQkkRcgKCFpGrZaI9gochOXYiCU8oY1KE,2477
+helm/benchmark/annotation/image2structure/lilypond_compiler_annotator.py,sha256=mvUg5tfJayACaqJW_wJOehZzBvLUlJppspphR9QYHOU,4034
+helm/benchmark/annotation/image2structure/webpage_compiler_annotator.py,sha256=rvzdQCaVFM6ovF28TSUnNmB47f2hidlaZm6vO4DJpso,6404
+helm/benchmark/augmentations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/augmentations/cleva_perturbation.py,sha256=arUkY_luc274YEMZocOos9rpAZVbEFZphbMlobAxTy0,29208
+helm/benchmark/augmentations/contraction_expansion_perturbation.py,sha256=yni1UR2fviN0Wig8MpOp0zzLn4H-gYocTjKTpxBwywg,4850
+helm/benchmark/augmentations/contrast_sets_perturbation.py,sha256=OvuntCLHYZoWHXO1JTixWmnZD5iQDNy_yYrLwgAO15M,3338
+helm/benchmark/augmentations/correct_to_misspelling.json,sha256=L44RiJXlJCa6zQzTLf0MFHCOhFyRDRKfLQNXH-n3XIs,213429
+helm/benchmark/augmentations/data_augmenter.py,sha256=Uk7rMDEgJGDoFyJLuOepjclBiNLB7Y3fATCH0HP_k_4,3847
+helm/benchmark/augmentations/dialect_perturbation.py,sha256=4DJB2gDj-JzhYtu-uxh9zgebChRDHpwMgKhju8OqP44,6317
+helm/benchmark/augmentations/extra_space_perturbation.py,sha256=D7y7oA4LwKJnVgTuLZOF3nQjmYRtkezuxY75qVQkD8I,843
+helm/benchmark/augmentations/filler_words_perturbation.py,sha256=NihLbpyxlXnncxM635pIyQ5LJvu4uPb4oKiLb8UU5dQ,2922
+helm/benchmark/augmentations/gender_perturbation.py,sha256=rnFREHD3FawmFaaQMsI-bhS52fhma7i1m8mZghX1hww,9387
+helm/benchmark/augmentations/lowercase_perturbation.py,sha256=e-lhkuKOV5QR_GnOp71vr2lHI97tiWSZ_jz57jcFbvg,519
+helm/benchmark/augmentations/mild_mix_perturbation.py,sha256=q27-c8_di1jaLhUuo5LzqOStnJcWIjSJ0a8Sr3S5TOs,1787
+helm/benchmark/augmentations/misspelling_perturbation.py,sha256=IE45qEE0HuIln62tAGvCP9B_RjNMJ5Xu06goL6b0fWQ,2144
+helm/benchmark/augmentations/person_name_perturbation.py,sha256=AIF5VVjNJ1OlUyCC1bt2dcP1KDn4gdgqd7Y2KGvHPqs,14422
+helm/benchmark/augmentations/perturbation.py,sha256=fsvyR2qxFqMKmxF1OCzVqmKkWa0FTk5Jff0wbgKHpDQ,3083
+helm/benchmark/augmentations/perturbation_description.py,sha256=VKOwBRPQY-0vuxhGvtac1Z5F10metPfpFnfs8ykFVmU,1184
+helm/benchmark/augmentations/space_perturbation.py,sha256=g4rbyoureBaOVf_lrRXIWYlL6YZpqbCOJk554iOCuIs,935
+helm/benchmark/augmentations/suffix_perturbation.py,sha256=P3AfJj_ajTVdjO7AJRQ9dKS-cT1PyRSt8Un57iZQDVc,785
+helm/benchmark/augmentations/synonym_perturbation.py,sha256=komOV5M342_8unopnwN6gkPWpJIZXidywiu6PO9_riU,4151
+helm/benchmark/augmentations/test_perturbation.py,sha256=ERQX6UySs5sep0lesNlQiPfdoTlRjFmHGTpg7vMxSWE,11902
+helm/benchmark/augmentations/translate_perturbation.py,sha256=dn8wO5UOgYbGtP9e77SmwaK2ginrQsTw-79nrzRzfeo,1054
+helm/benchmark/augmentations/typos_perturbation.py,sha256=_F9zwvrLie8hX7mzUtQmYq6oq6yqaFiKGsvc9LAuBr4,2798
+helm/benchmark/data_overlap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/data_overlap/data_overlap_spec.py,sha256=aj_l1l0qxUbUMrSWr70-Sb1j_JN-7WYop5BXPG_xj44,1998
+helm/benchmark/data_overlap/export_scenario_text.py,sha256=OiFsU_qME2_I87yDSNPfuAVI5Y9MbPEcEliYK6TaAEE,4527
+helm/benchmark/data_overlap/light_scenario.py,sha256=UFUr8plZD32e2TJTjFZLxTwD-ZRb9eYnHk2U3ZD8P40,1074
+helm/benchmark/efficiency_data/inference_denoised_runtimes.json,sha256=ios_dt-_8wtXvkVAx0iI2zwCxqHvk3XKTx31qHPalsI,4203
+helm/benchmark/efficiency_data/inference_idealized_runtimes.json,sha256=5w7reeZc0yc4cjH8kJGxQQSoe8yaRVX2SSlSrx0QWFQ,12348
+helm/benchmark/efficiency_data/training_efficiency.json,sha256=aH2moiBLStOLVi8Ci2KTK5ZkWlTBLK-B3fRfNZwhoSg,9763
+helm/benchmark/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/basic_metrics.py,sha256=7hk5PZL7d09uG1y7wHBhY_ox8hlXw-n7Yt_FDv_AIKw,20375
+helm/benchmark/metrics/bbq_metrics.py,sha256=Dqccr7GdfKNs1S_1QSB75d8AY7moovEPAqvacGfrCAE,6157
+helm/benchmark/metrics/bias_metrics.py,sha256=GQ4CwOk1Sa9g-LcJCxcoQLD1vWY2Hvujck9l-9qsmf4,11418
+helm/benchmark/metrics/bias_word_lists.py,sha256=mx5JjW3mHffXIqo4GcQN-zENUEttBqQnEjPTz3J3J_4,13909
+helm/benchmark/metrics/classification_metrics.py,sha256=uB23jRFzkmtJgs1sTO5pPjdV_mOg35gWubjGS8pynLM,5654
+helm/benchmark/metrics/cleva_accuracy_metrics.py,sha256=1eDxHxVk-JW1mF9SBcuplIefAoi_edUwKpp-XxYbmeU,2740
+helm/benchmark/metrics/cleva_harms_metrics.py,sha256=c_x9MYg8WjM1yym1S374GKxH_lwP6wZOiXrknf0mJis,11077
+helm/benchmark/metrics/cleva_metrics_helper.py,sha256=8UwiGhekUmp7DxYWU4rxqX2v3ewkg-O5-jOh49iOGmc,304
+helm/benchmark/metrics/code_metrics.py,sha256=e0aqLcxBAdCc0qAqebzK40Ilv2Py6xZbosud5v169x8,5121
+helm/benchmark/metrics/code_metrics_helper.py,sha256=h_y3BsYCbeh8cDe2LDndA8K6nkelV0J76qxlq2cBmsc,22334
+helm/benchmark/metrics/common_metric_specs.py,sha256=k_IW0A6BevAskS0_C6ZaP9XvIfrdLI974_NhC89rMoo,5846
+helm/benchmark/metrics/copyright_metrics.py,sha256=X9j3YsfzWEoGpgPpIvCzm18-JggLAW5QFooifE1KqaM,7729
+helm/benchmark/metrics/decodingtrust_fairness_metrics.py,sha256=TcyklpfcTMXrpJeaHQfxS9QQxe-gwmT-HD0g_DmIFLQ,3253
+helm/benchmark/metrics/decodingtrust_ood_knowledge_metrics.py,sha256=TxTkkWdx6d6ym0MirZTiucl_TWFdn4uJLnlTfLjQvgk,2925
+helm/benchmark/metrics/decodingtrust_privacy_metrics.py,sha256=l6TmAflBSgQGLjB-U7wmMCvY0Sui4ni29K7mkajsdtY,3412
+helm/benchmark/metrics/decodingtrust_stereotype_bias_metrics.py,sha256=KhiJ_7tnK8kHg2acKPAUWBQvAtyvjJTwcWJMUSUBHak,6416
+helm/benchmark/metrics/disinformation_metrics.py,sha256=BsLBG5krHGQh_nKDAeSDpp-li89s7fYoYTf5T0RZfMM,7782
+helm/benchmark/metrics/dry_run_metrics.py,sha256=d8RgltW4nGTH1tZeGOIlQRwRaJLIxL60h46NXv_wv1s,3674
+helm/benchmark/metrics/efficiency_metrics.py,sha256=Y3y-gVOgKfAcsWUPTINmlmyJ-mOyrQ22KuHvhMZPCRM,11522
+helm/benchmark/metrics/evaluate_instances_metric.py,sha256=EBUf0ONnNoi7pcxYab7RD0B_JqGksqDX8TOaosSmJk8,2847
+helm/benchmark/metrics/evaluate_reference_metrics.py,sha256=CtVqdmQBdHxXyb1Cv3UZwCstHTOjNPgQ0cgmEJ6mMSQ,15063
+helm/benchmark/metrics/instruction_following_critique_metrics.py,sha256=Pj1itUJi_KDy0D-FOPcOyHqm4ypHMfhbAVeDJzGlyeo,9773
+helm/benchmark/metrics/language_modeling_metrics.py,sha256=ofqwj1PMJQu16QhLDULXBmZ5iFz91ducwLRpNsRYELE,4510
+helm/benchmark/metrics/machine_translation_metrics.py,sha256=bp_EDXyxntIty5gORDa7va-C73quOzoTc5o8MpxFmL4,3816
+helm/benchmark/metrics/metric.py,sha256=dPq7ZMB0w-LgJKMzWYDJtfn-oYD4oG4jJX0yiUEziJM,14245
+helm/benchmark/metrics/metric_name.py,sha256=POhgmUqqIWh_LjCbYpiKkzGqqChBLeW3FADy9u_FcWw,1354
+helm/benchmark/metrics/metric_service.py,sha256=mlX_MEFSYNzME6GFS3El_VVOvzPYnOMosKI0XIxygP4,1802
+helm/benchmark/metrics/numeracy_metrics.py,sha256=panMWD3a1NPerg3Ix7l6NhR7jGOIQOQV9i_KysBeDA8,2818
+helm/benchmark/metrics/paraphrase_generation_metrics.py,sha256=-VkAknRhAEBmC_lpz_1aeXU8OppL8KfEPtIYCJkHTmw,1981
+helm/benchmark/metrics/ranking_metrics.py,sha256=5hDRapsxx_cmo-ag_80kOQnrgZn3lfVsLZVtWxuxH-s,17391
+helm/benchmark/metrics/reference_metric.py,sha256=RlIM_PFTEkBo0_EEMq8d4_BSagNSBR_XyovMtjDeqqU,6026
+helm/benchmark/metrics/statistic.py,sha256=FuxNxMtAfiCkOxBS9KHlhEyxe61e0YXt2emvsufgPZQ,3424
+helm/benchmark/metrics/summarization_critique_metrics.py,sha256=Lf7PDuce62HDzyofsyxaOvH0QvzcaS-vJvDWtIs8xKk,4694
+helm/benchmark/metrics/summarization_metrics.py,sha256=laLMGRDy1wjcFvgSWXvzOZwBXshkmPr0S2Ofu79Z01Q,16461
+helm/benchmark/metrics/test_bias_metrics.py,sha256=N22a5UKqSd7TfYBrZS6ciFSOIc1s85pYMwk7m9ufpG0,6274
+helm/benchmark/metrics/test_classification_metrics.py,sha256=8A7UqgSkJzVgQ0IIMfozL4HTrj8GiDFxsf4_dov3Amk,5613
+helm/benchmark/metrics/test_disinformation_metrics.py,sha256=U3ZmS9s33oimTQbKO-7pgWeX_WiDB9chlOCtf_vslXw,2249
+helm/benchmark/metrics/test_evaluate_reference_metrics.py,sha256=cRZwInR-XLr9Q-q_3d-TXk2kIx8RJQ8w15-_VSUS7HI,1128
+helm/benchmark/metrics/test_metric.py,sha256=kLP6oANAuvR-MQmZJJybN2DQGXYa6oQarO96a4QDhOg,749
+helm/benchmark/metrics/test_numeracy_metrics.py,sha256=ls1ZIHDePKpHMoqAbf4HmJ1SIBjLFuLIzGbfg6OiZvM,4162
+helm/benchmark/metrics/test_statistic.py,sha256=AejuYLSeUwEOqpEMRKZFjnxu4HKUraeExU8TPmZEqW4,1229
+helm/benchmark/metrics/toxicity_metrics.py,sha256=6MCpHuCXbXZqWwvO57ifKYHnHWBzszN9cZjwgPQQF2Y,4027
+helm/benchmark/metrics/toxicity_utils.py,sha256=-bfittLtMkHyV5wu-hj6KVtaiNGgVIO5duUmThBlX8w,988
+helm/benchmark/metrics/unitxt_metrics.py,sha256=5rw_fBQGWpFLr1nR4HcRlAwYvDZfJ6_MzGozzNo5NOA,3605
+helm/benchmark/metrics/image_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/image_generation/aesthetics_metrics.py,sha256=AXQjWBd9zBZOoCF8vQV9FjUy33teC0IF7pdbq-XiHjM,2101
+helm/benchmark/metrics/image_generation/aesthetics_scorer.py,sha256=ISdThDKMrx-SHQe69dCcr8qUrMCa_GsxX3BeZnd0WPA,2538
+helm/benchmark/metrics/image_generation/clip_score_metrics.py,sha256=tUnAoew24jjjbjPaoE2-4iyRTq6YNW8Xfk1p5JWZkAU,3338
+helm/benchmark/metrics/image_generation/denoised_runtime_metric.py,sha256=Nom_yw15ePU7wUuV2DFHpLnEAqaZQjlkW9LowRElOAI,1646
+helm/benchmark/metrics/image_generation/detection_metrics.py,sha256=n2SmrnQ9izucMjwD9KBThgVBo3_R5AGTZJXwqXdbZvg,2121
+helm/benchmark/metrics/image_generation/efficiency_metrics.py,sha256=neeNJNtHAVUMWqr5rvRIRlPKl225cXUGCURLB0z-rKQ,1459
+helm/benchmark/metrics/image_generation/fidelity_metrics.py,sha256=Vewml_NOcM2jK-yyKHWsHB0KC3NVG8HfweA4rGZ9RAQ,7583
+helm/benchmark/metrics/image_generation/fractal_dimension_metric.py,sha256=i71ACxshXW6zVI5D8UzkNtIU5hMrPQR6byV4cKJFVJE,2096
+helm/benchmark/metrics/image_generation/gender_metrics.py,sha256=j_sHhAkq1fA2VL483OX80cC9EQjzOIWGHQAeGVEN8fY,2371
+helm/benchmark/metrics/image_generation/image_critique_metrics.py,sha256=fVgb-GdA9QmNywq7byNlG1TTxb_zvzP_bcURruZ54Wo,12958
+helm/benchmark/metrics/image_generation/lpips_metrics.py,sha256=naVxg-yXdXum-yQD6MgRcemVr0L567Y2drGjfehravQ,3582
+helm/benchmark/metrics/image_generation/multi_scale_ssim_metrics.py,sha256=z3xaiKrfarF-lfQ8Aa-Dveaun3LfMMpQIWR8bHbQIrY,3507
+helm/benchmark/metrics/image_generation/nsfw_detector.py,sha256=X1hsWRBa-1KOhT_TTfCk9_jsXXMXHadAxddsFmTKQHc,3914
+helm/benchmark/metrics/image_generation/nsfw_metrics.py,sha256=ZlJfqsFh6DNFLBBTNBDB2UK85wlsxtsxTlN9k5_i-5Q,4626
+helm/benchmark/metrics/image_generation/nudity_metrics.py,sha256=KQG-jybgaWuXB5rL-kLBWBD7gdZKWdTJYYrAEEMnZAU,1553
+helm/benchmark/metrics/image_generation/photorealism_critique_metrics.py,sha256=0bwfJZ5pOAQFec4TEf_eUd9qQxwximwzDrYFkZnO9Yk,6951
+helm/benchmark/metrics/image_generation/psnr_metrics.py,sha256=VLq9gOkaoIZNAGII9fMI3tOCMpKAzbfLuqGbWo_mOV8,3126
+helm/benchmark/metrics/image_generation/q16_toxicity_metrics.py,sha256=g85Rtc6MJPO1dcjD_V1oK6-k91ES_GKmB1QKoVSZcnE,1924
+helm/benchmark/metrics/image_generation/skin_tone_metrics.py,sha256=0QXFXvG5bvBZ5eXVrj0N08Gz6xqlyPjZ3oqv0jD4OCE,6023
+helm/benchmark/metrics/image_generation/uiqi_metrics.py,sha256=NaBxJvZJdNw8g7Z6jxUC3mtUhdPXsib45TSK2fr5DUM,3904
+helm/benchmark/metrics/image_generation/watermark_metrics.py,sha256=GxxHK8i6WzCaR9YDYyn0mte6T9EcgyMy1BV2HtJQInA,1885
+helm/benchmark/metrics/image_generation/detectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/image_generation/detectors/base_detector.py,sha256=e4c8vPfioGzl2ftYzWOFIBDJcZJxBmpjU13n4fXaSvY,226
+helm/benchmark/metrics/image_generation/detectors/vitdet.py,sha256=fhjEs-iMlpKFAWw-qpKZS4wko5nn09Fm5SYt0FFUQzs,7477
+helm/benchmark/metrics/image_generation/fractal_dimension/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/image_generation/fractal_dimension/fractal_dimension_util.py,sha256=vFO6s8QHo6Pt1QfbOKAI0m3mJrc0BeH1Hcf7u2uWMIk,2116
+helm/benchmark/metrics/image_generation/fractal_dimension/test_fractal_dimension_util.py,sha256=O2EDENxDfDmbrL54N4ev2LdhouAaTh0xXP3NhNyW_ww,1415
+helm/benchmark/metrics/image_generation/q16/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/image_generation/q16/q16_toxicity_detector.py,sha256=8Y5h-6RMjivm50RnNbNwV7wCug4RhKT5g8R_YeEp54I,3467
+helm/benchmark/metrics/image_generation/q16/test_q16.py,sha256=f3Ycj7gGz1-tSxATvfYEGh1yyJwzcHbC2Yru73f3mPs,716
+helm/benchmark/metrics/image_generation/watermark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/image_generation/watermark/test_watermark_detector.py,sha256=__f7NVsVQatDFn_2Bfx7ObiQ68kAMvyyClApaTxqx80,649
+helm/benchmark/metrics/image_generation/watermark/watermark_detector.py,sha256=w6WnTc6t6zx0W0gTjgedXC9OO5dq5iWpx9UcnioKml4,3641
+helm/benchmark/metrics/summac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/summac/model_summac.py,sha256=zEuTI75eBBZPzJp0j2kFd2HejouhugC83nSWoVpghBQ,17412
+helm/benchmark/metrics/summac/utils_misc.py,sha256=7_Q1c72cKt8PWtxn8u4R8nB53HK6_JF2nP8bBXYNk-A,1485
+helm/benchmark/metrics/tokens/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py,sha256=WeNP4yiM4TVrD9Kid-uVRmWIVDqETnBsMycZmIBiTZ0,665
+helm/benchmark/metrics/tokens/auto_token_cost_estimator.py,sha256=MAVCme5cK_ryPjoA6cQPvfpKwqdSfnliTaxwT0vi4R4,1974
+helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py,sha256=5igmDhWu7H8-qiJnAQqCTQd1WXRewMrpdJdo4qhvSWc,523
+helm/benchmark/metrics/tokens/free_token_cost_estimator.py,sha256=G_6UK6Js_NZ_eqY0ZQnrC9QJVMERGhV1f6v7xq2lM-Y,461
+helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py,sha256=9zjtuxMbvfPBYuxOYMFEmNP8ZKFDVywrZ08n6nrjbA4,1520
+helm/benchmark/metrics/tokens/openai_token_cost_estimator.py,sha256=7jgjcgmbcVfLA_nTOrWXKIF8TEXng_KnE6cSgsSXWmE,1398
+helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py,sha256=l9UQZ0aAIhCYuFbIjU3j3A3XXoMvwUvz1kvRtlDbtOo,1079
+helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py,sha256=_wJ3E3LbJB9XPLixTH82BYQbp32o3oij6Sz3lsZL30E,2648
+helm/benchmark/metrics/tokens/token_cost_estimator.py,sha256=fTGUfhHV6yMwpTkCEMTGMxKO8jskqJz4sAtwXT6M_C8,425
+helm/benchmark/metrics/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/metrics/vision_language/emd_utils.py,sha256=3yN-DY5rxMabmtLV003lj59SRnp_T83sLAi96rycKEo,15043
+helm/benchmark/metrics/vision_language/image_metrics.py,sha256=8dtrEYAjYKtr1ID5ytjKxiTwXg0loppBjYAV8Av956c,20059
+helm/benchmark/metrics/vision_language/image_utils.py,sha256=XeYF3E6MnYyPJ5hYp4TtiTP27-y4S8LTBH5bZVcvJFg,3758
+helm/benchmark/presentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/presentation/contamination.py,sha256=PiIdcaD3-xfExjOmyL5q4Ao2ASa-OlScJAB9u1Zxe7o,2811
+helm/benchmark/presentation/create_plots.py,sha256=2-ZOuEdRwqqF1biRmzWggMZjmODoxOQOBoz9GT7tVww,28737
+helm/benchmark/presentation/run_display.py,sha256=tC1DciLvDTQJog4BDo8StWDdX7DbBkhrG2sX_SwXSPQ,11838
+helm/benchmark/presentation/run_entry.py,sha256=J1QgLOP99N7N4bs7nzXWxyU3pOd-a1j8xwL9ag1nP_Y,1158
+helm/benchmark/presentation/schema.py,sha256=pOwHCLvAC1Nh6vh48HV83gb7T7WREkifvo4qdovFdv4,8511
+helm/benchmark/presentation/summarize.py,sha256=2fJ9BYOJRxe9eBylLUK3qcZZwAwRtJF_C8plEQlAPEU,67266
+helm/benchmark/presentation/table.py,sha256=-foH1BIfMiD6YvpwoGJ910CH7Hib-_pYtHH1hE8zwNc,2904
+helm/benchmark/presentation/test_contamination.py,sha256=RlihBOF6vx2tKEj6_EMnJojTYoStx0FUeJSLT1bdf8w,509
+helm/benchmark/presentation/test_create_plots.py,sha256=5PPPegMTdBZurxyyUxI4rN13AVsjV3eQrwFqlobJ8UA,1286
+helm/benchmark/presentation/test_run_entry.py,sha256=OM-027j2A0Lx-ai2zBprOxSqzZhS_dh0OKw3ThocZW0,751
+helm/benchmark/presentation/test_summarize.py,sha256=UfSp33Q9xvuGnPYfFmLJdH5y7KWp9qbZprRMyx8LGP0,1618
+helm/benchmark/run_specs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/run_specs/classic_run_specs.py,sha256=YKrjuuPXoVjUT6XGEtyouPHkkE0XfR6u2xHQDRqpNyA,57972
+helm/benchmark/run_specs/cleva_run_specs.py,sha256=lEIHEqQY3Efx-sl2Z6Rq9Qq_1HEWHqFYuUkZbGvq66s,13387
+helm/benchmark/run_specs/decodingtrust_run_specs.py,sha256=D5g_--eFOI6-hy6fv9JNj_X4DHU03prKA5GZjlqaoRk,14254
+helm/benchmark/run_specs/heim_run_specs.py,sha256=Pt1eVbzvwZ5EXq8WB2b3XYw62SWYN_i1P_H3oE4i8KY,22096
+helm/benchmark/run_specs/instruction_following_run_specs.py,sha256=GElJhgbQhlZMYSAM4YyGcYq0pqycR32kBCoHqG6m-ZY,4177
+helm/benchmark/run_specs/lite_run_specs.py,sha256=ViCPJ86Aah8301GTEk6z4_MtP0g8iik33t4GudobhWQ,11113
+helm/benchmark/run_specs/simple_run_specs.py,sha256=0kK_e8U4JUWZ6wO4N-GPFRE1iGT4ilvSMUGfirvpIE0,3837
+helm/benchmark/run_specs/unitxt_run_specs.py,sha256=ejp_knrcIjf0J4WiKj9LTgDTcUr29-XFZYHYz0w_dkM,1518
+helm/benchmark/run_specs/vlm_run_specs.py,sha256=S3x80FbO7JQE3L6A0tZ70lBLKThfC0vAASdyP_cj3Ko,17477
+helm/benchmark/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py,sha256=Wyt7J5BAvAqC5JTqCW4fh7ex9-itX11P_9rLTocqvtk,4973
+helm/benchmark/scenarios/babi_qa_scenario.py,sha256=S1tPQY2x1I3hQL1JQ6wvUwvKyiSe7SqpRSW6N3_T0mo,5043
+helm/benchmark/scenarios/bbq_scenario.py,sha256=lT1XKSM-PXYtENI-ryScC4yb1TtII7YoH8kt_S1dZQo,9579
+helm/benchmark/scenarios/big_bench_scenario.py,sha256=bSk8Ia4u_6OqMjiyadpYQAWN-8GFWqvd3Ft3JiVGpi8,8081
+helm/benchmark/scenarios/blimp_scenario.py,sha256=o1MDcHT14KFDET4K9otx8pDiIgXrhsD19pvO0mR2ADU,6260
+helm/benchmark/scenarios/bold_scenario.py,sha256=NEfECMVzlVP_yo6sOuIzj6vZ5jd72_nvtEQ1lWrq85Q,4106
+helm/benchmark/scenarios/boolq_scenario.py,sha256=rvSp5SwXMCVzBo5BFxfhj1Xv06_ksqKrtTQR7nPiS-o,8013
+helm/benchmark/scenarios/civil_comments_scenario.py,sha256=VO5G-cQ9qctmBN0O76uSewnO_mFslMo5mbR2ZTrjuds,4851
+helm/benchmark/scenarios/cleva_scenario.py,sha256=xhwZ616iz0CN3fYIfrXHcV1XlcRQjyPSzML8fq8D3l4,57939
+helm/benchmark/scenarios/code_scenario.py,sha256=s4AGW8eBY0gFnu6EXvVWL0xbFYO28N9sgP1V8eBO7EI,12171
+helm/benchmark/scenarios/code_scenario_apps_pinned_file_order.py,sha256=KC-5MQ-d8Nn46aDN4FaPxmd6yk1DtVUmVR-CIZsNCp4,1738
+helm/benchmark/scenarios/code_scenario_helper.py,sha256=TnXAlY-wdAFwIDylFItf0z7HOu93WD6dNThwzZYe330,5904
+helm/benchmark/scenarios/commonsense_scenario.py,sha256=Wgry4P4W0hHmeBrN61u5h6VTjET5TZI0oozC2ri3nbQ,9487
+helm/benchmark/scenarios/copyright_scenario.py,sha256=qw-ijottZ28DjMqMWSZzL9_FuRDAHf0YmGV27YstmVI,3673
+helm/benchmark/scenarios/covid_dialog_scenario.py,sha256=LqNtfb9XQestluN0Bz7yi-Zo0VRkLjl4F9IOYzhPw3I,3971
+helm/benchmark/scenarios/custom_mcqa_scenario.py,sha256=aZkt1Ums-kIVpLfd4Pfhe7Rf_fZLxjO2ANLL2aN2qVM,1915
+helm/benchmark/scenarios/decodingtrust_adv_demonstration_scenario.py,sha256=hznH_Xe5vY269vtuApWwAaGUC9_1Pz9D-nfdG1a06cQ,7909
+helm/benchmark/scenarios/decodingtrust_adv_robustness_scenario.py,sha256=9qo3l44aby1EfQqohh1M2DVtHXqY1fuvj1XT3_n4hBk,5588
+helm/benchmark/scenarios/decodingtrust_fairness_scenario.py,sha256=rAOZnFSxO3ENOvcNz688P_f3Y7NzdwiWgoYTNvAaw3A,2866
+helm/benchmark/scenarios/decodingtrust_machine_ethics_scenario.py,sha256=qhzqW614WnsiyN7TiHUdZY_NpEdW_iMO0AMrLK8DmK0,14116
+helm/benchmark/scenarios/decodingtrust_ood_robustness_scenario.py,sha256=RSigvRdqjeFTwFfXNmslz8zyAGSmLf6UtBDA4NrQBCo,8304
+helm/benchmark/scenarios/decodingtrust_privacy_scenario.py,sha256=goGmHtN7MYnAQIXhffZZhuuuMWN0gHNOXyI9_injiZM,20119
+helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py,sha256=mbUABlGhpDur6x7z_q5iDqJRMBZ2d4ZI3KdVWNnJagM,2859
+helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py,sha256=AI8HX16_Lw9MKqrck62q8IFLUU-P5hxaOEHcmTS4rdA,2928
+helm/benchmark/scenarios/dialogue_scenarios.py,sha256=-I7FY6q1b11zpFd1_oAgar5qlfaFcXsNCKGVln9etPI,5629
+helm/benchmark/scenarios/disinformation_scenario.py,sha256=kQi0MVVoSDhx2vOTnUaCIttPXMf8zz7Eld2FD_77tnA,8504
+helm/benchmark/scenarios/dyck_language_scenario.py,sha256=vMxND9wPJenrGlCLhSw5UxOw3TV2Jq8cTmIXGpzEWaA,9318
+helm/benchmark/scenarios/entity_data_imputation_scenario.py,sha256=n2mnkmSeTznEy7S-GVumqpD9bt27yctbuEmtgQrG-Y0,6399
+helm/benchmark/scenarios/entity_matching_scenario.py,sha256=YjBX61TlL3CDQ3X6D-JyR-qlOYGLdoRXJxl9AEeqxYs,7022
+helm/benchmark/scenarios/entity_matching_scenario_fixed_random_state.py,sha256=TklbX7Kx4y-estV-YHUbI5O08q2qCZRrOmX9D3gZS9c,2193
+helm/benchmark/scenarios/grammar.py,sha256=Pb9vEP_0Ki87UdQCj1ym7QWJ24M4DRP6TXB5d3GnhLs,5597
+helm/benchmark/scenarios/grammar_scenario.py,sha256=bl-Cm9caDs077zSu38mzaS9maZ2gM-QazgjOEMFvxYg,1454
+helm/benchmark/scenarios/gsm_scenario.py,sha256=9fV2SEw3ocKNAD-TrDZZTpq4l7mbttQQWbO0YNz4e6k,2613
+helm/benchmark/scenarios/ice_scenario.py,sha256=vvk11cFPGUhg_CcGh3wEfVsGzrvMFgkByN-xcF-OOjI,16473
+helm/benchmark/scenarios/ice_scenario_pinned_file_order.py,sha256=fuirubIdi-rkJMfSd7YoDdBX2q0f5K7GGTN4XVapAUY,1613
+helm/benchmark/scenarios/imdb_scenario.py,sha256=X1k76AweFECCpYCXy8HuvjRbXbfmDfwK3SES_t_wkUs,6174
+helm/benchmark/scenarios/imdb_scenario_pinned_file_order.py,sha256=fjW0Gkzg2Y3IAbtYJ3KC7MueWd9U8h0tlcBCqxYmRrM,1621
+helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py,sha256=PuwcuHnx3nCuRYEEzEQ6aRI9i7QAHtfg4DmLAG4M6AM,2205
+helm/benchmark/scenarios/koala_scenario.py,sha256=ihx39W1C9KyOh5MkdMqw1twUwzA8S5QPsBKeQRZcZGM,1365
+helm/benchmark/scenarios/legal_summarization_scenario.py,sha256=DTYYsf-74NFjHvrNa5ZWSgMywq9ss16r5eFZI2-xZJk,8644
+helm/benchmark/scenarios/legal_support_scenario.py,sha256=hu8SlDGrU6oUTQjnrLEagjFNniW96wMahKjnrtuyjzo,3925
+helm/benchmark/scenarios/legalbench_scenario.py,sha256=55D09TaELT_B__ERyyWUp92gGwXifNgX1eIOzw2pgBE,4398
+helm/benchmark/scenarios/lex_glue_scenario.py,sha256=vsy8Y03hoCwTJFgBdQf0koxlHDRFm2V9wT04n2K1tlk,10268
+helm/benchmark/scenarios/lextreme_scenario.py,sha256=HEpXLdLLywo6yfXaGF3zRgx1PiGPE2ILnpKYs9Dmsow,20396
+helm/benchmark/scenarios/live_qa_scenario.py,sha256=meJvkkbqRUQ2odKkmlOLYOVVFd0LGDhGzK5nHCjXW6Y,3642
+helm/benchmark/scenarios/lm_entry_scenario.py,sha256=_7lTn2kfUQHi3ynX7fEaoseUYkpBx-ANn62kqHr-bY4,9085
+helm/benchmark/scenarios/lsat_qa_scenario.py,sha256=Ui68mi05N3WxGMZfCznJBscKkqaJ0ZEUO3PhIiD1xNE,6129
+helm/benchmark/scenarios/math_scenario.py,sha256=mL6Xf0xTAKcrH_YGqTWJxACnctGRUIQh7alLoLqUL8s,14144
+helm/benchmark/scenarios/me_q_sum_scenario.py,sha256=WLfED6DiIOUPd4PowydPmrQ4v26OsjG4iHumq7ka9do,4001
+helm/benchmark/scenarios/med_dialog_scenario.py,sha256=f7hTcEpxGfoxJ-WXNrSRrgVfbyuYpB624HbyRuMP-KI,7295
+helm/benchmark/scenarios/med_mcqa_scenario.py,sha256=JdVQuzKM4TRyWT6c1zpsMvNk1RYHnKdy_fRM6W7PCLI,5035
+helm/benchmark/scenarios/med_paragraph_simplification_scenario.py,sha256=e9WRx41XKJI06l3ZGkZe_LKNx-qd4hoWopfDncrr3KE,7591
+helm/benchmark/scenarios/med_qa_scenario.py,sha256=R0iDMzu3U0TnYEOiyjnlOWHGFwhvOJBVwJ5-Hos2eP8,4417
+helm/benchmark/scenarios/medication_qa_scenario.py,sha256=F7Pi_vYzFzKxAF0ruDaHE4soz32PjxeI6T_qxX6TeV4,2217
+helm/benchmark/scenarios/mmlu_scenario.py,sha256=wZjoGjaZNERyqSDHgIsv8fIANyUUQZ1ExQAa4cPzmM8,3790
+helm/benchmark/scenarios/msmarco_scenario.py,sha256=5DTWSDuinXVomlUTYfwOijXLifCSeG4be9Tvb7SANQM,34026
+helm/benchmark/scenarios/narrativeqa_scenario.py,sha256=FIzd4CTS6ZTOI_TJ8aPlkXYNfqbTIs13ZUqAt7ie8U4,5608
+helm/benchmark/scenarios/natural_qa_scenario.py,sha256=khKCR8xVhRqJtZoAfGQ9_CV6REkfrYn5ahLpci6rUzk,12540
+helm/benchmark/scenarios/newsqa_scenario.py,sha256=g7bFXWCTvRjofmmUJhx3jmCOvcVwi_P9LSVrNofvMLg,7260
+helm/benchmark/scenarios/numeracy_scenario.py,sha256=ED38H_AKxTFhKEJizhQXlfgkBColxjIdv4j9KvFI0Zw,30824
+helm/benchmark/scenarios/open_assistant_scenario.py,sha256=PH8F8zqYXXakr1xttBtFmENGMGciIpyijK4RCkJol_0,5495
+helm/benchmark/scenarios/opinions_qa_scenario.py,sha256=s0dGhsgcgud_bSqXw6p-w-nC_cme1Vjt9T9AwU4-K84,7371
+helm/benchmark/scenarios/pubmed_qa_scenario.py,sha256=zVL1gb3eVz-LbK2hfdnRR9ItaMSPlAGJorByWLt-4wk,7506
+helm/benchmark/scenarios/quac_scenario.py,sha256=SRAhMp6TAsmTRq6VRONLl3SEayFIe23He_mBhzkZ7qM,6628
+helm/benchmark/scenarios/raft_scenario.py,sha256=_5QhHS3opxxML7Rek6F-q5NVOf0M2UgbC6OTnQZ4C1U,4452
+helm/benchmark/scenarios/real_toxicity_prompts_scenario.py,sha256=GkgJo_13MWQQQTZbhlknvTR6ZrYr7NEn1WdMZrPs4y4,2400
+helm/benchmark/scenarios/scenario.py,sha256=lkJgqDAbnFQoJgAyAllqvlyIIH1nSQ3dnoVYh68tQrU,8232
+helm/benchmark/scenarios/self_instruct_scenario.py,sha256=jZ2MksT4N_4g_sp5egw7ycrsM-Ya786_RFmiYYdMvG8,2285
+helm/benchmark/scenarios/simple_scenarios.py,sha256=ersSzp9bFEFfpJ-SNy368AuonwswLnuyA1n7FOgkw4U,6459
+helm/benchmark/scenarios/summarization_scenario.py,sha256=MlNMgsY369DC04nhMUdG2o9Ydi6yze1fGOjC0bK-UwQ,6847
+helm/benchmark/scenarios/synthetic_efficiency_scenario.py,sha256=pzifpsJJbucmTjujNqQnwQa4Y7wpQjkS6QjNXOrgTAQ,3096
+helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py,sha256=1b3e3WpFMNBV3li17-0Ug6QCSKO4qRFaWDF23bYNsvQ,16326
+helm/benchmark/scenarios/synthetic_reasoning_scenario.py,sha256=k8IGK6VABOr6wuha4HynP47peoAkmIViAVhScOtCANo,8345
+helm/benchmark/scenarios/test_grammar.py,sha256=sPlA36sHpThbXgnGlXyOuqHfDPe2epIafmzIeL0nkoU,1364
+helm/benchmark/scenarios/test_math_scenario.py,sha256=s3-CllgCB8DL9-L4DmJ6Zcf9xi803nWYN84KlhN7PhM,1016
+helm/benchmark/scenarios/test_scenario.py,sha256=HexTZBKphMDJbhIYj-HRCDwltPTDqHFHdT7FjPmu8Xs,2070
+helm/benchmark/scenarios/test_simple_scenarios.py,sha256=9b-gtuRnd638q_JevVlEVsHzMZSzOe8j0FrUQmMyZM4,1736
+helm/benchmark/scenarios/thai_exam_scenario.py,sha256=FinZuwEz5dDcNBxG4OseVOnBWlgg9lT1LzMdIWSjG94,5838
+helm/benchmark/scenarios/the_pile_scenario.py,sha256=RqU8yXQJ4FkmEc6rO9J3QMXenyUZrsEZlLAQUx4-Wnk,4995
+helm/benchmark/scenarios/truthful_qa_scenario.py,sha256=iqL-tuqUQZjF9-DKAnI6wV-oLGC_I3aFuofdtJYHt8E,6035
+helm/benchmark/scenarios/twitter_aae_scenario.py,sha256=CRlPxVfkg3HPZV-lUMyCUSFOiAqg5IIPt-dq3qR9LU4,2096
+helm/benchmark/scenarios/unitxt_scenario.py,sha256=adz6mNQFQgSbEpiOPePGMXh9aynABovHoCW3wWYA_wQ,1692
+helm/benchmark/scenarios/verifiability_judgment_scenario.py,sha256=AaCq9n_YC1BJ_cjNlR9Mtm_6it_eu14mtNCPbiBC5rI,6043
+helm/benchmark/scenarios/vicuna_scenario.py,sha256=zLwLuEr6n9VQjVxQwgFIM-os23kJewr4Uiq_1zPiScM,1661
+helm/benchmark/scenarios/wikifact_scenario.py,sha256=cOVKgDisBdjPcmVMCLhTekdgX3hpDJUT-aKbvRSaMoM,5791
+helm/benchmark/scenarios/wikitext_103_scenario.py,sha256=PAPkmZdC4aIBQ1k29dDvTFBEFaPV1ZR1Ifif4FHoZqs,3087
+helm/benchmark/scenarios/wmt_14_scenario.py,sha256=NArkTZntYdYlegHo_-fkzeyCUOjosOONQKlquPbZRxY,4498
+helm/benchmark/scenarios/image_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/scenarios/image_generation/common_syntactic_processes_scenario.py,sha256=c8zcoGCOFqBGE4TAEx1uLsUmGXw_jIS8alI99ubGeDA,5477
+helm/benchmark/scenarios/image_generation/cub200_scenario.py,sha256=7p3G4mJRc8QHR4Mw2GLsfAFuJcEe6OeZbezVhbyc55E,4103
+helm/benchmark/scenarios/image_generation/daily_dalle_scenario.py,sha256=yiaX_2Aut3hZdfggCsTCcTPOdraaX-cOjPHyE5D1lYg,9045
+helm/benchmark/scenarios/image_generation/demographic_stereotypes_scenario.py,sha256=yAFh8Kn84b3RpHZVsVMYnk4GvbMgxB7oIPUzv5sp78I,3216
+helm/benchmark/scenarios/image_generation/detection_scenario.py,sha256=E4tqQXmPSLutPYdk9Ngil7LteUwRe7p55MXSURxk_rU,3188
+helm/benchmark/scenarios/image_generation/draw_bench_scenario.py,sha256=b4WcbttcjoIY5gcOMk6e7c_mqfjlQsfowo9-D_-wccY,3179
+helm/benchmark/scenarios/image_generation/i2p_scenario.py,sha256=8VXVyWZmW6k4Q43GBDwqzYDj1pCKuAraGKaulZ_0MPk,2233
+helm/benchmark/scenarios/image_generation/landing_page_scenario.py,sha256=0R1sAuS7txLxpaJOuvojg6dZEcRFGo4WabjX2ieo0FA,1361
+helm/benchmark/scenarios/image_generation/logos_scenario.py,sha256=Sbc_vlMdHAr6AhJ6rr4TjBIMEbzcw_hdGhYGufHSlJc,10245
+helm/benchmark/scenarios/image_generation/magazine_cover_scenario.py,sha256=swcdlPAYYwtidMUIHWCW1-xxHH2QF3wm3IKDBkr6-DU,3599
+helm/benchmark/scenarios/image_generation/mental_disorders_scenario.py,sha256=kybgeSjY0nm3Wya2vQeoBtJGtHZra4XhT4VY7Izjxrk,1879
+helm/benchmark/scenarios/image_generation/mscoco_scenario.py,sha256=KMzu0II1F8ajL7NrYBWe74UXKcLugg7bhubpSZR4TQw,4201
+helm/benchmark/scenarios/image_generation/paint_skills_scenario.py,sha256=6nm_uo6rmkyBzbU0BntvLq9jhWlfunBEy9Vc1d7_RVU,2967
+helm/benchmark/scenarios/image_generation/parti_prompts_scenario.py,sha256=pTSd1XidvWwTPFpVinl28s6150a2w3iK4d1Ce81lijA,5165
+helm/benchmark/scenarios/image_generation/radiology_scenario.py,sha256=7JN8OYap8kA36Od1_bZTkhZd-H9Qjw_dh4TIzr9UTms,1701
+helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py,sha256=DoabanZhd-2MHFDZeR9EoPit0T2TvbVwZGUR0RfJyW0,2362
+helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py,sha256=IB4_GbzQjjXBp-551XZ6PTNUCRX1jLcGfB3bVFI5lo4,3547
+helm/benchmark/scenarios/image_generation/winoground_scenario.py,sha256=E2xPQNQzylDSmqLjjMkQB8D7A6g7bzqtSF4bXPgfVbI,2889
+helm/benchmark/scenarios/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/scenarios/vision_language/bingo_scenario.py,sha256=LiH14xUoEKXn5ZStDbGE4bz9iMEn3-5I39eJ6kvN2UY,4045
+helm/benchmark/scenarios/vision_language/hateful_memes_scenario.py,sha256=us75yZygU7_xPrNLi5yvRBx3G9O3_HOnRRgfjcbqXzI,3709
+helm/benchmark/scenarios/vision_language/heim_human_eval_scenario.py,sha256=7GK_jAOfCgRIGiN_GInDePwuT2wZqmWHp1rqdx18xQg,4994
+helm/benchmark/scenarios/vision_language/mementos_scenario.py,sha256=Yw4zxeYgUw8HKRR5ob9QEIT0bSPrdRUULMKCo_xzFpw,4337
+helm/benchmark/scenarios/vision_language/mme_scenario.py,sha256=zxtdub2akvxPYEG12pkW2c57TIFqN38C7ucAXAHAdx0,5455
+helm/benchmark/scenarios/vision_language/mmmu_scenario.py,sha256=XQv7uv2m6EdbI7h0-9eDag4_bL7qE_78PuHB7c4SsHA,7654
+helm/benchmark/scenarios/vision_language/multipanelvqa_scenario.py,sha256=HuizbYsN5Nlihfzu4bfGuC8KSBbeIc6TVknMS4kpVJY,7149
+helm/benchmark/scenarios/vision_language/pope_scenario.py,sha256=uFkzMMsjhmuSYo3v_QdfJFX6RFse83JjzMfMa3ynvV4,3975
+helm/benchmark/scenarios/vision_language/seed_bench_scenario.py,sha256=5MwGb9BOyB2Xy70BGYZcjencf0ZskxBuzcPa7ABRuww,5106
+helm/benchmark/scenarios/vision_language/unicorn_scenario.py,sha256=e3lCq2nevy9tIFDDKEbJvmLibfk4UMQtAIyzrgnnaZs,4179
+helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py,sha256=VJCTQU0jG_ROtGD-fu21BBrcMGHH1Fv-_DTEzEOHUjg,4127
+helm/benchmark/scenarios/vision_language/vqa_scenario.py,sha256=vAf6MkYNDgi6_ODhz-rCY_OVqXKxGVJiwsYGnUw7H18,5089
+helm/benchmark/scenarios/vision_language/image2structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/scenarios/vision_language/image2structure/chart2csv_scenario.py,sha256=ImhfiC_y_hihAGvlj9zRsaoW614QFCBopBD2KxnbSs0,1805
+helm/benchmark/scenarios/vision_language/image2structure/image2structure_scenario.py,sha256=-eWRwo2x7kR46Z_I4vFbVlbqA_1f2UEb75Dx84XTlNE,9028
+helm/benchmark/scenarios/vision_language/image2structure/latex_scenario.py,sha256=FKKybU4IeglwXCj6GZC8cAUs_GOU7ymEa6P1dkDT7uw,1350
+helm/benchmark/scenarios/vision_language/image2structure/musicsheet_scenario.py,sha256=UvYzQzAvWRrydPty4LEifCjo502xNaHlmRVUdduvk9k,1048
+helm/benchmark/scenarios/vision_language/image2structure/utils_latex.py,sha256=ovg8-FfJ8_I1xbajFGSLvERZIA1fQjaUn0zd04ZbI84,15316
+helm/benchmark/scenarios/vision_language/image2structure/webpage_scenario.py,sha256=PSVBqNxvE5GXEsa3iBuK51vzp3Nho25Gjv9KL_uOMLA,9628
+helm/benchmark/scenarios/vision_language/image2structure/webpage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/scenarios/vision_language/image2structure/webpage/driver.py,sha256=i-i0mlG5oRRDNYNqP7o7Ul56iL02p_anJoThXaSvFiM,2826
+helm/benchmark/scenarios/vision_language/image2structure/webpage/jekyll_server.py,sha256=9WntahzuhVv54IH1m7_z0IxwLma3dbaMOne_pUx751Y,7652
+helm/benchmark/scenarios/vision_language/image2structure/webpage/utils.py,sha256=JpTiCSNcqX4wRpH6Cl07jM5wMkpZ5OeR_hjfK_V-Qok,943
+helm/benchmark/static/benchmarking.css,sha256=7PsUCff8YcoVxyWuALItfB4TZL55T7MbVQpoc9nZznc,2104
+helm/benchmark/static/benchmarking.js,sha256=lqEmoAikBwycVBf1h-et3ZmHKW_DcwxzlwmDez2A1EU,54531
+helm/benchmark/static/config.js,sha256=kIfkgr6gaMdFOAdqB35EvuBohq0DWYSQZbe_pTK09VM,103
+helm/benchmark/static/contamination.yaml,sha256=rAfh1DqwyUcDtyzHPQ2QiUK5eY7QfuuRtBXpZMn4TeA,3171
+helm/benchmark/static/general.js,sha256=qcsntanG5UMWK2vznSVAVFy9zd3BMc8DFfNa7KKezew,3053
+helm/benchmark/static/index.html,sha256=xIJGjMg0qn9eemfdBiNbTI0jzPfBD5x0v8HJF-dMqBc,3561
+helm/benchmark/static/info-icon.png,sha256=P-PW3Ek3NGiRAW5BXOjJRPBfMVqprjAqtQheGWu7zNI,3428
+helm/benchmark/static/json-urls.js,sha256=AaULgfHw8OLfrQLJpBHfcC013uavQnlNNFS9vzb0qOg,1981
+helm/benchmark/static/plot-captions.js,sha256=bTR8gYx-QqF_RJyKX-L-eQP7hSEtawfJSoADCvgjKag,3011
+helm/benchmark/static/schema_classic.yaml,sha256=p-yc2WMfyGehRtD7L5ZZHbFMMQovu2HNfvct3tBlV2I,108168
+helm/benchmark/static/schema_instruction_following.yaml,sha256=mg2g5P8TAYSCEhZbLfshPt_Hq2GKjwbvyOsQrwDqh7w,8923
+helm/benchmark/static/schema_lite.yaml,sha256=62ByEWhAJT0tIUFi-euxJ7XFhE6e9E6PT9dF6V3qoSU,40255
+helm/benchmark/static/schema_mmlu.yaml,sha256=8kiZDEGGaBXs9ucDk_Gbo2agV-OgOmWuhcYFyodRjcw,53307
+helm/benchmark/static/schema_unitxt.yaml,sha256=89GnKrooG7kKU2xh0MeoYZUB54FDUAmOPrbzuBhG1Ik,15496
+helm/benchmark/static/schema_vlm.yaml,sha256=IJLEPRe9ZCqVPRLQSIc-VpuUmZx03AJ3r_K8C19_zNU,21823
+helm/benchmark/static/utils.js,sha256=bgN0PT53Dregc-nLmEmAEmg2psufWpS8jTf74WoypHw,7681
+helm/benchmark/static/images/crfm-logo.png,sha256=dDkauL_wJR_Luu7L7pltphS3a9HSLjDkpVLa6C9vcA4,62712
+helm/benchmark/static/images/helm-logo-simple.png,sha256=LtVAC4OgcWgMAob53rTrf7cRDu-O0z85ZOGGj9wR9hw,86133
+helm/benchmark/static/images/helm-logo.png,sha256=GTqbrxJr0oQXbBRq-8v6afY5zB5x0M6PhEbKRIX9qIE,280667
+helm/benchmark/static/images/language-model-helm.png,sha256=mG0-bkdziXeiF0wOGd67y2jnYmVKJYqhD2N5Q8VIF8Q,26563
+helm/benchmark/static/images/scenarios-by-metrics.png,sha256=F7g9mvIYopm-n7sDGg-7I0XCyZvloKsi2wIq1i6da_Q,51331
+helm/benchmark/static/images/taxonomy-scenarios.png,sha256=2MiuCLaxnuHvwsWWJHnZFc-rvoQIi_tNIjDatY7I-Dg,100766
+helm/benchmark/static/images/organizations/ai21.png,sha256=Drkew6Vlwi2_4_S8hjagK2x8smOwLKTNiXIT3rDiurs,10208
+helm/benchmark/static/images/organizations/anthropic.png,sha256=cNi8OdIshIIb8PdodcX8mAj-khaUD0O6nhah-_6nYfs,8017
+helm/benchmark/static/images/organizations/bigscience.png,sha256=fwQAwN1x2Fr_ztD_HZdcOkdFcyxuDjtS3B5-VuRNkuc,19036
+helm/benchmark/static/images/organizations/cohere.png,sha256=7cr4LI8WK9yPryQboyWK_T5baSND-d-tVrlPNflLQMg,8757
+helm/benchmark/static/images/organizations/eleutherai.png,sha256=uUURFF8YWY85mwGoKVEjArO5DUBCy4es5naCXsBzn6c,4526
+helm/benchmark/static/images/organizations/google.png,sha256=BtmXrVQZHr3WH5c8c23ent2FO8aPWeNwO8czl22lDCo,4914
+helm/benchmark/static/images/organizations/meta.png,sha256=VYDp8arkAe2eYRJhAOcIAsZY1qY0hqyOEQDgVMbX9M8,4646
+helm/benchmark/static/images/organizations/microsoft.png,sha256=9e5QFl23yTbnAk8u7lZKaQOf4oPHbr_aiQda5n4MZqE,50850
+helm/benchmark/static/images/organizations/nvidia.png,sha256=hvp1wZMwYxkfrVMvJs73PX71JwY5L8ZvxIH_fL4n6Po,27945
+helm/benchmark/static/images/organizations/openai.png,sha256=P4ZT5ISIlt6Dl0mOp7juSM4Y7dfyRNPqdc0PJuwNoqg,16877
+helm/benchmark/static/images/organizations/together.png,sha256=pmWjW4r7GnlKqFhKLPTiBeILiOighL3XzcSCsxWtB7U,48053
+helm/benchmark/static/images/organizations/tsinghua-keg.png,sha256=l9SzlZCsLF18BY876wYJcVgiQbgvwte7uoILPDcVwHk,7776
+helm/benchmark/static/images/organizations/yandex.png,sha256=OOCdcKubAP4x7h4VW7z5a-AHPWBiSDTjsIJea6ZiovA,27964
+helm/benchmark/static_build/config.js,sha256=ER8utDIqVZi9uge7Qrk1gmlT88TOOkFF9xYp3j10m8U,165
+helm/benchmark/static_build/index.html,sha256=fqZee8nuPxeRbBNzPgY0AUmGFHF-ngY1dG_MfwfZGLc,1149
+helm/benchmark/static_build/assets/01-694cb9b7.png,sha256=aUy5t0DYCg4r52HDOmeNi1S2CHsnv3mE7ySokJg3Ouo,8903
+helm/benchmark/static_build/assets/ai21-0eb91ec3.png,sha256=Drkew6Vlwi2_4_S8hjagK2x8smOwLKTNiXIT3rDiurs,10208
+helm/benchmark/static_build/assets/aleph-alpha-7ce10034.png,sha256=fOEANHS8RymKaCzUWn9gQWebts2ghSmtW9Fdda_TjR8,7224
+helm/benchmark/static_build/assets/anthropic-70d8bc39.png,sha256=cNi8OdIshIIb8PdodcX8mAj-khaUD0O6nhah-_6nYfs,8017
+helm/benchmark/static_build/assets/bigscience-7f0400c0.png,sha256=fwQAwN1x2Fr_ztD_HZdcOkdFcyxuDjtS3B5-VuRNkuc,19036
+helm/benchmark/static_build/assets/cohere-3550c6cb.png,sha256=NVDGy09xliCqZy2TKUAka-B90jVDB_VRCS9A2_sN7VU,4414
+helm/benchmark/static_build/assets/crfm-logo-74391ab8.png,sha256=dDkauL_wJR_Luu7L7pltphS3a9HSLjDkpVLa6C9vcA4,62712
+helm/benchmark/static_build/assets/eleutherai-b9451114.png,sha256=uUURFF8YWY85mwGoKVEjArO5DUBCy4es5naCXsBzn6c,4526
+helm/benchmark/static_build/assets/google-06d997ad.png,sha256=BtmXrVQZHr3WH5c8c23ent2FO8aPWeNwO8czl22lDCo,4914
+helm/benchmark/static_build/assets/heim-logo-3e5e3aa4.png,sha256=Pl46pKbC_TU3L6kZQ_3G-0wTseluAhIYwb3EqpdQAjQ,1344452
+helm/benchmark/static_build/assets/helm-logo-simple-2ed5400b.png,sha256=LtVAC4OgcWgMAob53rTrf7cRDu-O0z85ZOGGj9wR9hw,86133
+helm/benchmark/static_build/assets/helmhero-28e90f4d.png,sha256=KOkPTf-q28PdvGOBp1G5O4q1eWUJjuij3z2h_SUUf8s,55314
+helm/benchmark/static_build/assets/index-5088afcb.css,sha256=UIivyx7XvQHnXD0s-J7WaeOZKDl2mgNxUts8n9D57Es,486112
+helm/benchmark/static_build/assets/index-d839df55.js,sha256=d0xZkf7R2yZMo7pQTJb9J1dqZRg5hcqaaoFutsjocbQ,66843
+helm/benchmark/static_build/assets/meta-5580e9f1.png,sha256=VYDp8arkAe2eYRJhAOcIAsZY1qY0hqyOEQDgVMbX9M8,4646
+helm/benchmark/static_build/assets/microsoft-f5ee5016.png,sha256=9e5QFl23yTbnAk8u7lZKaQOf4oPHbr_aiQda5n4MZqE,50850
+helm/benchmark/static_build/assets/mistral-18e1be23.png,sha256=GOG-Ix7XlctGOUmvJfO2oVSBM7E5O562G88OnoxsjBw,14402
+helm/benchmark/static_build/assets/nvidia-86fa75c1.png,sha256=hvp1wZMwYxkfrVMvJs73PX71JwY5L8ZvxIH_fL4n6Po,27945
+helm/benchmark/static_build/assets/openai-3f8653e4.png,sha256=P4ZT5ISIlt6Dl0mOp7juSM4Y7dfyRNPqdc0PJuwNoqg,16877
+helm/benchmark/static_build/assets/react-d4a0b69b.js,sha256=rNTpl8Is3LkYXqJowRMc8vc4SXQwP94Ozy4DZZWwldU,275141
+helm/benchmark/static_build/assets/recharts-6d337683.js,sha256=rDrVmtTCCSLY2hpcxSDxhlQ6CQmTTSQOESNeO3oVQgg,432466
+helm/benchmark/static_build/assets/tii-24de195c.png,sha256=JN4ZXAa0rbR2IlxPfd_mKtntFZcYpDcXocSiqrC2rNg,63389
+helm/benchmark/static_build/assets/together-a665a35b.png,sha256=pmWjW4r7GnlKqFhKLPTiBeILiOighL3XzcSCsxWtB7U,48053
+helm/benchmark/static_build/assets/tremor-54a99cc4.js,sha256=x_K5Bp7szI2zsvESrKqffUOHbm8ohjjvuoIeY_yD_CA,293015
+helm/benchmark/static_build/assets/tsinghua-keg-97d4b395.png,sha256=l9SzlZCsLF18BY876wYJcVgiQbgvwte7uoILPDcVwHk,7776
+helm/benchmark/static_build/assets/vhelm-framework-cde7618a.png,sha256=zedhimhku2Q3QIvaRSYlUAQ0b5ia9pU4cFzKnABfr4c,118544
+helm/benchmark/static_build/assets/vhelm-model-6d812526.png,sha256=bYElJoVkSaMJ_lFZj5qoSrIbygbNyBk35q89jtFRet8,168494
+helm/benchmark/static_build/assets/yandex-38e09d70.png,sha256=OOCdcKubAP4x7h4VW7z5a-AHPWBiSDTjsIJea6ZiovA,27964
+helm/benchmark/window_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/window_services/ai21_window_service.py,sha256=1ZDLJv73bxoLj_MzEBu4TgH5xHw-hx0nI6KX6RU73kE,12593
+helm/benchmark/window_services/cohere_window_service.py,sha256=5jm8o5ZYrbDUluA5LbMWLOOrOlIuHR7MhAJkOuzBagM,4750
+helm/benchmark/window_services/default_window_service.py,sha256=F099qF-YeM7YPVtph0dRFPry5vP8_BiudHTy2CpuICQ,151
+helm/benchmark/window_services/encoder_decoder_window_service.py,sha256=EU3QevFOiQYBN2te54FsVRnGYZdgDxK6KqOWSQOa0q4,2125
+helm/benchmark/window_services/ice_window_service.py,sha256=9NeBN_tmOvwrK1miUnX3wJA70BP5ifIIeHpNR2gVwls,1070
+helm/benchmark/window_services/local_window_service.py,sha256=e9JHG72kFWlV6UKg_IhRCipOWQUrOD6ZjsT-_Mwewps,5232
+helm/benchmark/window_services/no_decoding_window_service.py,sha256=s_i_cqIuU9p0GDRIBApaOHzjH7gHrBPTJ2X5NEcN33Y,1375
+helm/benchmark/window_services/test_ai21_window_service.py,sha256=HkpNSaJAClZfaa-bQZ2BrRm1UB_u4sLAGSBlGQqRUD4,8221
+helm/benchmark/window_services/test_anthropic_window_service.py,sha256=lnxLiW5BPaWN6m03L93qCFugsxnVBbLmYPCarlrO-So,4196
+helm/benchmark/window_services/test_bloom_window_service.py,sha256=x7WBh0S223ABC9KvL2-y9G-cUxFUPm6oIkqvYO_4mt8,4288
+helm/benchmark/window_services/test_cohere_window_service.py,sha256=rKXnw2E7MLAtkLgtrUvnZuQp99_agDO4qcpb3daik-E,3348
+helm/benchmark/window_services/test_cohere_window_service_utils.py,sha256=sf25f9MeXzoqsbDzZ7d7le13hm8RkDe54nhLtKF2pqo,158150
+helm/benchmark/window_services/test_flan_t5_window_service.py,sha256=IhQMWBq2d39O3uNKGwbaMWJkz8585Zc-J_yqvPJfwu4,695
+helm/benchmark/window_services/test_gpt2_window_service.py,sha256=2UHKt4Wmh6XmSCdepjuMbZHFpb1oUcrKRSxcdOzBE1s,2671
+helm/benchmark/window_services/test_gpt4_window_service.py,sha256=tV5WdpxYxewchEp1rnsIlEfdJFrHVFKYQ-_8NhGK2yo,1052
+helm/benchmark/window_services/test_gptj_window_service.py,sha256=0lu4Os_3x3N-AbejG3LZ3-_ikxEHg1Lbmfq-Pzg_D9Y,2374
+helm/benchmark/window_services/test_gptneox_window_service.py,sha256=8CaOW_ln9bxKA4--dVLfLdsASo6RrR7ouP6EcSruzdA,4210
+helm/benchmark/window_services/test_ice_window_service.py,sha256=1DudvCYh4te_UDLg14XeXwKUoin9QnCgZ_PSCwxxaM0,23579
+helm/benchmark/window_services/test_openai_window_service.py,sha256=W_QJKaMgzYU7qGFuSS6JeM_f50UX0SuHpkH-u2bEvI4,2312
+helm/benchmark/window_services/test_opt_window_service.py,sha256=Gh1GzWnlgYIGwDNBw4EnHds3fXwMaSjzkfFXeLn47os,4215
+helm/benchmark/window_services/test_palmyra_window_service.py,sha256=yy7D2C0ZzExCbptYNsEI9zuX2AEGsEUTj0a_vbqub4o,4212
+helm/benchmark/window_services/test_t0pp_window_service.py,sha256=pvp55FyqjunkDpHVAhPup3h-iNkepQpxyr4nC87-5iY,3998
+helm/benchmark/window_services/test_t511b_window_service.py,sha256=zwgUxmkpV0IJ-SW15Ugic08KEx-ZwLXPCl91Mt87XDk,4001
+helm/benchmark/window_services/test_ul2_window_service.py,sha256=JUehWFC5P1sosoFzPacLJwZQ3D7_GUn6yLi8oBPkido,4061
+helm/benchmark/window_services/test_utils.py,sha256=TQ5Ba3rq-0k9fCqGJ-gfEr2NavP_hzvxwZ42SRPfIlI,3376
+helm/benchmark/window_services/test_yalm_window_service.py,sha256=tO1ZsUCXD1E0QZWarABJjn5Q-g-d1PyM8PFOmGMcynI,4301
+helm/benchmark/window_services/tokenizer_service.py,sha256=dPsEhA1I4SUvsR0UBdzJ2wsFahw8GuF6qhE8CxiPZRY,1051
+helm/benchmark/window_services/window_service.py,sha256=y6BthPY1V-ugmYfaJElm5Wfy3PSgoJLj10vHcXZZGNA,4727
+helm/benchmark/window_services/window_service_factory.py,sha256=T55F0Y2jiOYxUHHZxT4YX4fFXY5gfFhn56zIwUBhc7s,3423
+helm/benchmark/window_services/yalm_window_service.py,sha256=_Yz4NwbMx9Px8raJlMMA5Aw80iA8G_bQnd8pxRK-By8,1059
+helm/benchmark/window_services/image_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/benchmark/window_services/image_generation/clip_window_service.py,sha256=2JHld8GiR_eIQyHMPSN8K2VOswmKJEPMPJLsxlLpU-Q,631
+helm/benchmark/window_services/image_generation/lexica_search_window_service.py,sha256=g7usb-xefJtUXpgywm0cnCgYYAHq5qTX3o0V0tTQGQ4,407
+helm/benchmark/window_services/image_generation/openai_dalle_window_service.py,sha256=GFz7xf0uuN7ah7QC7SFq2833QHvefNJvBu5L310MAfs,406
+helm/benchmark/window_services/image_generation/test_clip_window_service.py,sha256=domn2MRduHVAdruSUuGPDIGKyDrh-gFxW-fZaBYR7cg,1430
+helm/benchmark/window_services/image_generation/test_openai_dalle_window_service.py,sha256=nSyKK-cQxZnase3Bw4X6DyAWZEy1OZi4stDZpKtolF4,1411
+helm/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/ai21_client.py,sha256=LIdkmzcUDR9uIF2tIk5YgDNGNmfQ9JDYmgscvFoCHDs,5509
+helm/clients/ai21_utils.py,sha256=mlg3h615kyckccGZv9rqsP4Y60O3XpwyE-UURRMrxII,471
+helm/clients/aleph_alpha_client.py,sha256=koPqXF6uRD905atoiCaPg5yxr6B25J0g2OTWk8geebQ,4969
+helm/clients/anthropic_client.py,sha256=woFek17fbUJ30vVdVj1cZX8AxI0VwMoc5eevu9z69C4,31585
+helm/clients/auto_client.py,sha256=Qs0XFq9pyH4M9HTOLoI3_5m8kW305x3pzVukgETdrZM,10732
+helm/clients/bedrock_client.py,sha256=BsH9UopsP6ZHf-K0Yzg1PYSMLDwY0yIUmPHDhJVMUi0,5293
+helm/clients/bedrock_utils.py,sha256=okZ6Z8pviGOUNlrdF2QquAqFs8-QYgcqci95eij8giM,2574
+helm/clients/client.py,sha256=xoxPwV-aar7suM-3eAMsB9FmrempyqZ5FFXcLIQJz9w,8628
+helm/clients/clip_score_client.py,sha256=ct3GHZ2Zh3fGwyvQ9DyoIPT6PwDPI-nUaFkUFuc8PIE,1622
+helm/clients/cohere_client.py,sha256=0UUsFnHwZjEkKjXKPzM6EpZ_iuAduZTg3sCrPM1zGt0,7359
+helm/clients/cohere_utils.py,sha256=aYmj60m0e9RF9BIdxp1vmA-uZv17TEALw0dbgTUSpCc,504
+helm/clients/gcs_client.py,sha256=1sK5x5uWtThgz9gqBLaA8oyiXGD_9nn1WyfMzJRyPQ8,3231
+helm/clients/google_client.py,sha256=EOpPzK5_9yzWkMjK-4ILiixDF3aeOa8AbR2SPnEO-nw,2900
+helm/clients/google_translate_client.py,sha256=TgiQEscjOae58Ptgp9f4n0LXUtl1Jf6v9BI-Z1_wcuw,1304
+helm/clients/http_model_client.py,sha256=DBgkVDZPmg99DCcO_1Xdf6nFQo2kyxLkgoQpwC-wkHI,2806
+helm/clients/huggingface_client.py,sha256=vzUmNJKsgIXLD8ho4kUGyFCRFGXC61C74X7No0yY7N4,13235
+helm/clients/lit_gpt_client.py,sha256=Sjec16bNODosEhDoBkRc4t-LNS-nCUY_jVivWj5zvfU,6205
+helm/clients/lit_gpt_generate.py,sha256=8DdBE9ReQ00NbV3KMFYc--PlO9X-HMOR0Rhm5CADWEA,3103
+helm/clients/megatron_client.py,sha256=KFL1BBBDqxr5mtd5iu0dA6uK8_v6d4g_D6RsZrHx3a0,4107
+helm/clients/mistral_client.py,sha256=thOLMcEfrzWR00JUabIZ_PnW2o9YZsdSmNf9z3jbYKo,5982
+helm/clients/moderation_api_client.py,sha256=I5pYWRb2MmcLDYrScnC3P5N7OUFzQiVQ828_hf7zjM4,4719
+helm/clients/open_lm_client.py,sha256=qFgYqlV_3UiW8WJKz66lLqRqg2jt1qtJ1bHMRAtBn40,1749
+helm/clients/openai_client.py,sha256=ghwmaFCUTy82HeEq49g-PD-Z9hRs89bS_T0JMlzHgEA,13922
+helm/clients/palmyra_client.py,sha256=LBYFHNc5LdpPbiSp1AAHuMm8cUUCQ2EB03BB6XnDTYQ,6551
+helm/clients/perspective_api_client.py,sha256=WQDArqlKVWwcK2SicnSIAgV6JGVHsxibTzkdezT3z_U,5920
+helm/clients/simple_client.py,sha256=55S_y1eWD1bjktcG21Vs8G5bF6QbKKwmJyqs6lCUJeI,2048
+helm/clients/test_auto_client.py,sha256=bc-rsMJ8JM0MFnQ4B48hBJ1jL3RtRyVvmPwOgzF2mF8,3155
+helm/clients/test_client.py,sha256=g29C1WLUONnNuE2oGFZhaqMahb-doS4l_Ph4OHrQvrc,3895
+helm/clients/test_huggingface_client.py,sha256=WUPrA7VT3nnMNht7w20I6411hlpIS_77XbQC2vC0WU0,2723
+helm/clients/test_simple_client.py,sha256=G0JRQX69ypQN2VxhlNQXs5u2Tdtkcl_aeHqudDUVKi4,702
+helm/clients/test_together_client.py,sha256=lAtGKn3WdsYe5MEfTYVYRnu_rS4DPnfFr5jRn42rvoQ,3865
+helm/clients/together_client.py,sha256=xvqSaXvGeiJtf8c8MzgCMrXWM6m18y_kMFcry7NsmL4,12819
+helm/clients/toxicity_classifier_client.py,sha256=AI_FizxMurubTIyeceRdkixSnhWQbcD-oEEONj5ve7o,464
+helm/clients/vertexai_client.py,sha256=RMIuOXi7W0PGcZKy5ClsPc4dPVlVh6DHOCthQlUyrvo,18564
+helm/clients/vllm_client.py,sha256=p9atBtq3PBOoPkOPSifkMrYZjNLnNM_sWM6tL_3N-WY,1675
+helm/clients/clip_scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/clip_scorers/base_clip_scorer.py,sha256=NfXe79g6M4Wype3Xf-oXxscaUgjbZLmy9dRnBaLiWwk,695
+helm/clients/clip_scorers/clip_scorer.py,sha256=waLI_rI6dQPjmtywvGeQKK7bGCWXyoIgIuBc8P3zSB8,1907
+helm/clients/clip_scorers/multilingual_clip_scorer.py,sha256=7ntUK2XiQSnlrjjlNO0ywI3LNSOBvMLtWELewBKdSjg,2041
+helm/clients/image_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/image_generation/adobe_vision_client.py,sha256=-jhhKtgD8cV12fqoY5ws5SbOnlNX00lvgUtfpn9X6Eo,2863
+helm/clients/image_generation/aleph_alpha_image_generation_client.py,sha256=6CQCF1vtPgr5hMJAHrRYLNCM9lniYv1udZu0sy-8Hec,3999
+helm/clients/image_generation/cogview2_client.py,sha256=1sH-18x_Lo3h8r5gH4kT9p3vzfU_ughxQkSsLFoIKi4,8426
+helm/clients/image_generation/dalle2_client.py,sha256=ztW1BeTw1FaUdAMJeH-kihIOTgILmoTP8Zuuxe9mEN4,8401
+helm/clients/image_generation/dalle3_client.py,sha256=kVdP1jTTEWKnEFJ9wZWCTjQjP0X7vjjFoLCtLKfHviA,4581
+helm/clients/image_generation/dalle_mini_client.py,sha256=9JS_4w1egJlFSG_ACoxAlPiXY_mCi-bN8yXG3vaPxHM,8146
+helm/clients/image_generation/deep_floyd_client.py,sha256=i2Fp-jk1tXGjpXqGU39w3i-g3zNz1KlXEL-JWojMRKw,2999
+helm/clients/image_generation/huggingface_diffusers_client.py,sha256=LiX-0XbHV0GY2BMIwRdZcFN0saqEfWNzzXKDDY7VnL8,12284
+helm/clients/image_generation/image_generation_client_utils.py,sha256=N130PbHLLvE9Q1iVefPvTCJzs3hG3osZCeYdJyjLjCw,437
+helm/clients/image_generation/lexica_client.py,sha256=9Xpb3Oc0aaseZ1eAM0H3ShlgVy_UpFtuLXs74vD1zWU,3652
+helm/clients/image_generation/mindalle_client.py,sha256=kze8F_-mrJN2vVeq9fxSdWd0Ho0KES__CHH2zpDt2vA,4624
+helm/clients/image_generation/nudity_check_client.py,sha256=TeFga6HvBKgdX7LitBoioXUD4BQGavVwzr5BFFE29x8,2599
+helm/clients/image_generation/together_image_generation_client.py,sha256=T3HiE6em0Wwq_ilVThuCVWyej2jpK6Z4osw1M-Twa9w,4364
+helm/clients/image_generation/cogview2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/image_generation/cogview2/coglm_strategy.py,sha256=P3NU3Z4jsj171PrHPtGDiCRq05kEh-KHjSTgxPDw6R8,3766
+helm/clients/image_generation/cogview2/coglm_utils.py,sha256=EJPOEQJInCDVi2LHqkjEUsgw6GgVlLDrIptlT9cXk-Y,2900
+helm/clients/image_generation/cogview2/sr_pipeline/__init__.py,sha256=OsI4b5Lh0cuP7mDCCdMxzNVATEJk5A9S2unE2u21_Ko,338
+helm/clients/image_generation/cogview2/sr_pipeline/direct_sr.py,sha256=Vv6WDClDPAbwIW3S5qWObKm2hBGI8h9aIcHtVa5ZMbA,3603
+helm/clients/image_generation/cogview2/sr_pipeline/dsr_model.py,sha256=xYn3acxU4BRdDeRjk98Vj0qq8qqty93kPCLdz-bOMKs,10818
+helm/clients/image_generation/cogview2/sr_pipeline/dsr_sampling.py,sha256=OonYjdtNKJo12cNb-t-gFHLXRFxItCXjKgS9YxWAI-k,7718
+helm/clients/image_generation/cogview2/sr_pipeline/iterative_sr.py,sha256=dPz15YXG-XCzWKTduCIU7FYA2uBNyd9g-YUNz2L-iMk,5007
+helm/clients/image_generation/cogview2/sr_pipeline/itersr_model.py,sha256=5D1QWyAcY0CpwITk7EBN6ylUtc7mvZaE9iHG628AqMQ,10390
+helm/clients/image_generation/cogview2/sr_pipeline/itersr_sampling.py,sha256=d8voNZnXfqh-LzIzAqcMD4h4XJx_u_mvu9mewMrJC5k,4392
+helm/clients/image_generation/cogview2/sr_pipeline/sr_group.py,sha256=7Dzpf0iSGgVJFDaVgnPd8tPyPbKuFsy_Z0m5Z7NwO7w,1457
+helm/clients/image_generation/dalle_mini/__init__.py,sha256=KL0VUc5S9ygXUk06LUt6vJMV8BY2YTGtQZKFw2DtCDo,72
+helm/clients/image_generation/dalle_mini/data.py,sha256=UpiI2uWpwr9Eqxk8Al7iOZ6XcQgl1EawwVGpqvmvQjM,17582
+helm/clients/image_generation/dalle_mini/model/__init__.py,sha256=hRuE_LlrKedI68aXWN7DrgiVvd5PYqLxrbnZ8M7p6J0,198
+helm/clients/image_generation/dalle_mini/model/configuration.py,sha256=c-v-2ZUo7XX80txnWz4gduLcxmS-phYcWDBuLLJ5tNk,7815
+helm/clients/image_generation/dalle_mini/model/modeling.py,sha256=9SFAtImIr2IPBJt3BHG5ou-74oPG5S1MRbtL1kFfrr4,69610
+helm/clients/image_generation/dalle_mini/model/partitions.py,sha256=_fDpk34GL6NhNecHuP78y_gmKpWjbfw3fxMCWVEO4pc,2721
+helm/clients/image_generation/dalle_mini/model/processor.py,sha256=nTZMHRApm_PaUdQLhcBvxLI0BE_MfuQZ_pDkx-1MMZs,2222
+helm/clients/image_generation/dalle_mini/model/text.py,sha256=Kfba8JdO2LrSmCVlQtgc7J2kSordCgjeg7WV9V45B80,7302
+helm/clients/image_generation/dalle_mini/model/tokenizer.py,sha256=iu6YS1oNMEkamClQNxbsNlnYpEyC8-2bS7wyBnku-p0,199
+helm/clients/image_generation/dalle_mini/model/utils.py,sha256=clu2IiIpAT0DzTc2HvmI0ySnETFsJtpi7tocPkqOreY,1171
+helm/clients/image_generation/dalle_mini/vqgan_jax/__init__.py,sha256=4nOBVLNyIYohjXzZup9l2bSWdxEWZ1kckQ5ct7VKcJ4,17
+helm/clients/image_generation/dalle_mini/vqgan_jax/configuration_vqgan.py,sha256=4q39kdTUxeW55SN8NNkA9MdFZtH6rWssN8XauuOwyi0,1213
+helm/clients/image_generation/dalle_mini/vqgan_jax/convert_pt_model_to_jax.py,sha256=RicCxKNF7AUeeXYxpkPahN3Aq_aMqM3yAM8igdq48GU,4512
+helm/clients/image_generation/dalle_mini/vqgan_jax/modeling_flax_vqgan.py,sha256=O6FWL9jmzEa0sbP4Qc6E7gmwaEL9L5lbrbQ1_zQ8CCw,21017
+helm/clients/image_generation/mindalle/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/image_generation/mindalle/models/__init__.py,sha256=D7flcmyXa26YG-C7KNx6ovUu-p0tb4Je4suy7RZ8MD4,8127
+helm/clients/image_generation/mindalle/models/tokenizer.py,sha256=NFFdLUhoxEkv9SZqU3QIFk0ukaCcn6w_xFWQIRGhZJ4,1190
+helm/clients/image_generation/mindalle/models/stage1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/image_generation/mindalle/models/stage1/layers.py,sha256=cg5c6KShCMbMmHFuzDBIG-WgIBBkDrG9XvXC1DxqowA,11044
+helm/clients/image_generation/mindalle/models/stage1/vqgan.py,sha256=NCQSrFN-Ojknu-nt6Z1CDweQvL1P_thbsyf6ejJ4X4o,3983
+helm/clients/image_generation/mindalle/models/stage2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/image_generation/mindalle/models/stage2/layers.py,sha256=LvDADun5nMaencaRT0pm-dq78xHpPPkpi8rlu7RLHco,5306
+helm/clients/image_generation/mindalle/models/stage2/transformer.py,sha256=WJCvWUuY-U7nQT_6-KOB9phd1Q8IgsJy2O0f4Al2Vj0,10335
+helm/clients/image_generation/mindalle/utils/__init__.py,sha256=QnQ8JdgxsV89J7XOEGUO0uZEA-R_87_zua8XZ7wEdHE,67
+helm/clients/image_generation/mindalle/utils/config.py,sha256=lh8dXvL7ctKmuYEbeTQZfXN-_DkHQLjGuFuvo53u5pM,3234
+helm/clients/image_generation/mindalle/utils/sampling.py,sha256=soTHaJrN4FV1lDdh9HMveJs6F49UMK57Xfa0ccnHqI8,5029
+helm/clients/image_generation/mindalle/utils/utils.py,sha256=ESugpzG-_73GKl07mj-8o-_nim_FOICxfYkczy3s9x4,3119
+helm/clients/vision_language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/vision_language/huggingface_vlm_client.py,sha256=AMjOlg_Fb65Ah1d3XAt9M-842huhO_vOA8HQfNkAApk,4433
+helm/clients/vision_language/idefics_client.py,sha256=WQSDcoLNcVNmoKZ3PsAsIZOeZHiYQ33IXFz2-ktgVIk,7680
+helm/clients/vision_language/open_flamingo_client.py,sha256=CkN0JCeR742ZG9Nc4A85hp4BSE0WLU-3Rs-ZwdmDkzs,6632
+helm/clients/vision_language/qwen_vlm_client.py,sha256=6rCH4gJMDyQHyjAE_GDIrLsInH_bvd6to-4RMWbRLeM,7407
+helm/clients/vision_language/open_flamingo/__init__.py,sha256=i1tGJj6ckeE6eS1EWV5tbQKYLmPCrdSI45mPchfv_Ic,88
+helm/clients/vision_language/open_flamingo/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/clients/vision_language/open_flamingo/src/factory.py,sha256=1gFPR-hu520Jdzl2EWE9LSSlF-uuSOHoX3CLWUT4YWs,5652
+helm/clients/vision_language/open_flamingo/src/flamingo.py,sha256=0tn9uAyUbNVk3XDgZLndRRf2RN_Cy3qGOooBB51r0YQ,14661
+helm/clients/vision_language/open_flamingo/src/flamingo_lm.py,sha256=pUKtQ0gaR68S7IMR7M-NNlgg-MscsyIZ-FammqlX5Z8,6225
+helm/clients/vision_language/open_flamingo/src/helpers.py,sha256=pq_BgkUflYBDw8gxTO2evuiqvjw3bE9rx06iYHp0kTw,8595
+helm/clients/vision_language/open_flamingo/src/utils.py,sha256=6FYU0NgshZadF3QYWQkPW8jyEFiOd6jyb8p5rv_vOj0,1444
+helm/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/common/authentication.py,sha256=RlMx29_TSrfU7ujE7dJkxmFub5EqLj2NswV5lAVFFDk,179
+helm/common/cache.py,sha256=hPLBtWjCKlzccmfWZv56F6LEcLskkfLeq2DVHPeV2GM,7631
+helm/common/cache_backend_config.py,sha256=4u5A6BHNBmGnnrDNhCVgrdwhXQtyAbWcUeoo7hdgZSo,1530
+helm/common/clip_score_request.py,sha256=WnNg89owDCmG7tyy8nnQL0RdKQLsUdMWiYH9XqqbGw8,840
+helm/common/codec.py,sha256=gTh6AwIQ0Bbul_QSnIO7eItwMZmYtnkIrG1jkc4GOL4,7100
+helm/common/concurrency.py,sha256=8THtHlCtXo5c8iCuz_UcBBdzZX6aiEALLc4u0M4SYL0,856
+helm/common/credentials_utils.py,sha256=O-57nUgkWLbZF0k3lsSaVGPPHj2_OYeVuCMe0to3bRE,1118
+helm/common/critique_request.py,sha256=Exu8Ans05zCU5d5-AglEbG40mBwKYED2Z3WqY_XjXBY,2772
+helm/common/file_upload_request.py,sha256=OZeAW1_zsiNdXnWDwNNvhPs0b48TUmW_e4kzzCYmyiY,543
+helm/common/general.py,sha256=nMfHNPXyAAorAMmgDClD8r8XXeJcvfF0QXTP-FgH5PQ,11690
+helm/common/gpu_utils.py,sha256=pmLq6ipYNLEm28VxxSNeZuVt-gAw-WnYmBvxP1P1p6M,480
+helm/common/hierarchical_logger.py,sha256=EnKLnfbQftca08EJfjGEQb4tcnCKbx-JtwLnoCnhMQs,2908
+helm/common/image_generation_parameters.py,sha256=nsbuk_-BlRMK6IwP5y6BnTXbTRTOcvZ6uLblL5VHLOo,916
+helm/common/images_utils.py,sha256=uYqxp3nnfDwzg0hAiFX58ZPat944xKekLU_pWSwadXQ,2319
+helm/common/key_value_store.py,sha256=iHi1WQuWttLNJnuM48QNOAXHoneNbmbBmtXYPq-dyys,3147
+helm/common/media_object.py,sha256=3VZqfb0py5dDKwWtnLp2kdl8svaike-Cn7Mjk-b0cvM,5130
+helm/common/moderations_api_request.py,sha256=3xTsErSsCr2PHD2jpdV1JglHaYHwP2Yqu25_JFtfa68,2234
+helm/common/mongo_key_value_store.py,sha256=yK1qyh1RgKB_hYMD1BA6hQw6oGJdrALPMpqqlkn7h0M,3811
+helm/common/multimodal_request_utils.py,sha256=GNZQQCcwsARyFCO-uoeeglyK2PEfC4MjClAKDeKqokk,1404
+helm/common/nudity_check_request.py,sha256=VMsujI_RBy5u_cGEk0teE4KyX1dL2Zt3Pb4U6LpBdSY,728
+helm/common/object_spec.py,sha256=_usgTDQULBF6_jy7C6m-9ZNVvNxbGoTE_CdGcSvBASU,4327
+helm/common/optional_dependencies.py,sha256=Qam3QCHff8tuXbS-fCw-MVe-pK18gSvHw-uQoXXxT7M,616
+helm/common/perspective_api_request.py,sha256=WAVwtajNVmi5XJNsPcorGEAVrqkpPSk-Kd3b0hJghbA,2427
+helm/common/request.py,sha256=B94Dey42OJZ5lgcf71KsGW2nKo8eB4My_pj6tDkIQOg,8012
+helm/common/test_cache.py,sha256=XqboYHQAkFWIHPsuIjuageRSLeN7QoATKF7wwxggPqE,7054
+helm/common/test_codec.py,sha256=igL--k-2DwAy0eoMr8D9Xs8MOjBoT0LutbMPzDlTNkM,5885
+helm/common/test_general.py,sha256=c8Lh0mK8I-SfcMprq909B6zWRBxSBngq2nNL1L6-cYA,1788
+helm/common/test_media_object.py,sha256=AAm9DD7MC-ZvTwiqXA-e52U6L4S1noxItW8f7ARi6DY,1650
+helm/common/tokenization_request.py,sha256=1e-uCXUqF3ai83wgX9yV8yXPT5GuCTjJcLk-PszDlTM,3525
+helm/common/file_caches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/common/file_caches/file_cache.py,sha256=QfF1hlF8FQ-rcPn9Zyl6L0dOCokvYgd-dFqx4ftRuPA,359
+helm/common/file_caches/local_file_cache.py,sha256=wBOAbbkGLiClaX4YdunokRfSQCKNkTYmMVx2KTLy4Lc,1921
+helm/common/file_caches/test_local_file_cache.py,sha256=bOCWR9MglwQXV98xk8auyjgFxaOr85zRdxWwxMBQW9s,663
+helm/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/config/model_deployments.yaml,sha256=ptZz4bzZPbbuSCIvpyNm3UoKXZXxGsHyuAD0Kir9vH0,71211
+helm/config/model_metadata.yaml,sha256=LkHebb_FklRvm6DJCOO2leAcbeVQwfNI4xwcIuvMcfg,106991
+helm/config/tokenizer_configs.yaml,sha256=_XdrvaiBCPF09joBt6xKxuh9IIz3oW40whhThTfpx-c,11841
+helm/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/proxy/accounts.py,sha256=gd5cKhKeqklf_cXCAISl65AUvZeD6afBNrs6WK3IBvQ,14764
+helm/proxy/cli.py,sha256=l8F7UYqrIOoBD9ZCIxJFA4fhxlzhae0-2Nn8A7FMkzk,8244
+helm/proxy/example_queries.py,sha256=uYc05CIhTzFCjLPdkWqtOweyHqsRUX4s6ByP8wJbjVk,4650
+helm/proxy/query.py,sha256=eftbiUICMh8QIHVs-7cLtv_rDXKeKdRPmwjLMu0TDxQ,645
+helm/proxy/retry.py,sha256=iLZmKATEJQa9jsSpOIx6YDRhmrA8G1Qm21cUxCuo2Ug,3490
+helm/proxy/server.py,sha256=V05YdMy0lZqYfYkxLDqksGYe-8CIFa6Jg8aSb8YHM7I,10753
+helm/proxy/test_accounts.py,sha256=Vs1iOzTPN29LosDAAEs6IagQ3PccvutrJTlR1qNIcj0,1146
+helm/proxy/test_retry.py,sha256=8h398auzjW9VnlTJWllxR-bdpub-XFp8EN8LWDEnEHM,1049
+helm/proxy/critique/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/proxy/critique/critique_client.py,sha256=ATZuXw77lejwtpgLg3Soy3VDyv8D8xetl0o4guxDM2M,1764
+helm/proxy/critique/mechanical_turk_critique_client.py,sha256=OcppmFOMweBSfVTiLIICIwjvPpHHTkdu9fFUTaubitQ,574
+helm/proxy/critique/mechanical_turk_critique_exporter.py,sha256=taULrc_cIP0O9c5UpGz3l9DmWQadTVzN_v-qzTgMoyo,8470
+helm/proxy/critique/mechanical_turk_critique_importer.py,sha256=NL97joO5pRkcICRdVyG4kf9JhfYRaySsxRoZ7KWDYv0,5581
+helm/proxy/critique/mechanical_turk_utils.py,sha256=mKpUv4zz3s5ptzDY7UrwuI7Cr5HmNgSjPC10BnN9AL4,1766
+helm/proxy/critique/model_critique_client.py,sha256=nrNjnvOFdcRk9tUk2MjoBugAfMM92X0hxKGSg4xsy9E,11187
+helm/proxy/critique/scale_critique_client.py,sha256=B4povtceyfal95eE3N7em9cC_B5Vy4jMrHXcsXc_5m4,15889
+helm/proxy/critique/surge_ai_critique_client.py,sha256=HnzgAoF4Du9Me0GS_lbNaozZslS4a2OZx735gh-coo0,8357
+helm/proxy/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/proxy/services/remote_service.py,sha256=emYN0qWOJLQ7q1n06V4TwlvXaqylQcUxmqDcGZXqPJ8,9097
+helm/proxy/services/server_service.py,sha256=ehKs1gITG8ZsPpxzjbzlHqWjAJVLahiKZn5odsLhcPM,11535
+helm/proxy/services/service.py,sha256=Be-Z5F6AN4vMzsJr3BS6tJ9NHHy_dc_yn2Ex9cm0ChU,6193
+helm/proxy/services/test_remote_service.py,sha256=NFnLjg3QNHoDKdK0DlcrtylwlKXx1vdzheNZRrLEv7c,6605
+helm/proxy/services/test_service.py,sha256=FUZoI8pGiUg5adgB1wTJ869QOgFYjPtM6yf6FGMdE64,8968
+helm/proxy/token_counters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/proxy/token_counters/auto_token_counter.py,sha256=34PWvF96DurTrUtUxW4Td5VNV1_BhAebCkXQLl3xp4M,2046
+helm/proxy/token_counters/test_auto_token_counter.py,sha256=lDe1lXa5keRi1iLsKz2aBtoQyQ1fycYymZcGvshWvUk,8609
+helm/proxy/token_counters/token_counter.py,sha256=TCij1Cp08RoFTLLLdjNPoaeDGHpA1A2hQsrRV775Kf4,425
+helm/tokenizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/tokenizers/ai21_tokenizer.py,sha256=oXImuAY9kMohHH6Zm7BWysfT88b00NBoSELeGQ920y4,2255
+helm/tokenizers/aleph_alpha_tokenizer.py,sha256=UlWC_SjObBvexpZ3OfKZT2yjhbSsHlKjQe_oWuRrXno,3818
+helm/tokenizers/anthropic_tokenizer.py,sha256=d-HO9OEFkhYzFZu0VkOsHjxbqqSUseCNX0KQqgb3s2Q,2114
+helm/tokenizers/auto_tokenizer.py,sha256=xKL_rLnjiaCnyH5oJUlo5gfdVSen7PmBFFD60gl9R8A,4217
+helm/tokenizers/caching_tokenizer.py,sha256=kSegrCFotRevSDgJsn0g52dWiSUCNa7_EZpRNrELeUE,8163
+helm/tokenizers/cohere_tokenizer.py,sha256=-WuvEKHzwqcpnhDPauw7x8wyZ5eVWTZalygx1LkkLnQ,3739
+helm/tokenizers/http_model_tokenizer.py,sha256=wBTtDA2UdEYspffa1wqgkT3y3YHoyLXXoucnJ5PGjhs,3109
+helm/tokenizers/huggingface_tokenizer.py,sha256=IY9RxJ3YwVKfXtvMXR9DLO4uTaz9j_8hr1MOyA60H7Y,7791
+helm/tokenizers/ice_tokenizer.py,sha256=4ZTIRpmt2cqwcxnmrDpCRhiJ0BI3ELE-GHoBuHWgrDA,1200
+helm/tokenizers/lit_gpt_tokenizer.py,sha256=LMrpaje64UmnDKoYjPG_RQeXVA4xQUwW5t48IJIeLaQ,1660
+helm/tokenizers/simple_tokenizer.py,sha256=6_NROqVbygs-HRA7bYAZluN4YB5gUhVaRsYQeRTjA1E,1147
+helm/tokenizers/test_anthropic_tokenizer.py,sha256=_wzXp9FVR2Ml0s2A79TTXbSPHyTRp28i9tiEyQ9S6Ko,3792
+helm/tokenizers/test_huggingface_tokenizer.py,sha256=o1oqYT2MS-7xrnffj48WuvJfKAHd4p8pee9W4WxwQb8,6172
+helm/tokenizers/test_ice_tokenizer.py,sha256=-xi_f8TBSkAYr5CcA56HDq7rZ9HAGd99J7twNfkLzFU,2619
+helm/tokenizers/test_simple_tokenizer.py,sha256=vUNdcnJqZV99-E8H1rwUH85AQPJ2HTnDr5DrZ_-zRL4,1219
+helm/tokenizers/test_yalm_tokenizer.py,sha256=qWpKnUuAlePd6t-UJB_mAiBwtAacnC8caKXLJ_GdTkk,2477
+helm/tokenizers/tiktoken_tokenizer.py,sha256=FU2g_FF0pVoyspYhHcz3SyCBGNbsTby-nWVrj0Cq4_c,1265
+helm/tokenizers/tokenizer.py,sha256=1zi9fhgrs2znINKDnoEhW-6UatsVj30hRjQbTDbOdZg,1583
+helm/tokenizers/vertexai_tokenizer.py,sha256=lf-xckyeachaJI86ujNC2Cp_AVZ-BBcFgLZQy4lg7aA,4105
+helm/tokenizers/yalm_tokenizer.py,sha256=TBA6tlDH4A4OBKjghYfKxI-VujIhkQP83nsDgWF_x3Y,1451
+helm/tokenizers/yalm_tokenizer_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+helm/tokenizers/yalm_tokenizer_data/test_yalm_tokenizer.py,sha256=W9p5QNn1GSm-y85yVEQe_82zn5CVK_vR6jvhk7JTs_k,869
+helm/tokenizers/yalm_tokenizer_data/voc_100b.sp,sha256=LmPD0_OIOXi8dWuNjXUYOSPhf8kPp2xhvK-g3bXcwrQ,2815034
+helm/tokenizers/yalm_tokenizer_data/yalm_tokenizer.py,sha256=kH5Qig1_6r_sKbAHinX7C83tqBUoTwbe-gGZCbGVkko,6389
+crfm_helm-0.5.0.dist-info/LICENSE,sha256=bJiay7Nn5SHQ2n_4ZIT3AE0W1RGq4O7pxOApgBsaT64,11349
+crfm_helm-0.5.0.dist-info/METADATA,sha256=KyvhONCTx1d4n_igPIOM-GhnFjbXoqWkYQjSUM1P3G0,18973
+crfm_helm-0.5.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+crfm_helm-0.5.0.dist-info/entry_points.txt,sha256=AvH9soAH3uey9xffisWewd0yrmPWGASC036jHd1SFyg,300
+crfm_helm-0.5.0.dist-info/top_level.txt,sha256=s9yl-XmuTId6n_W_xRjCS99MHTwPXOlkKxmTr8xZUNY,5
+crfm_helm-0.5.0.dist-info/RECORD,,