ScandEval 16.12.0__py3-none-any.whl → 16.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. scandeval/async_utils.py +46 -0
  2. scandeval/benchmark_config_factory.py +26 -2
  3. scandeval/benchmark_modules/fresh.py +2 -1
  4. scandeval/benchmark_modules/hf.py +50 -12
  5. scandeval/benchmark_modules/litellm.py +25 -15
  6. scandeval/benchmark_modules/vllm.py +3 -3
  7. scandeval/benchmarker.py +15 -33
  8. scandeval/cli.py +2 -4
  9. scandeval/constants.py +5 -0
  10. scandeval/custom_dataset_configs.py +152 -0
  11. scandeval/data_loading.py +87 -31
  12. scandeval/data_models.py +396 -225
  13. scandeval/dataset_configs/__init__.py +51 -25
  14. scandeval/dataset_configs/albanian.py +1 -1
  15. scandeval/dataset_configs/belarusian.py +47 -0
  16. scandeval/dataset_configs/bulgarian.py +1 -1
  17. scandeval/dataset_configs/catalan.py +1 -1
  18. scandeval/dataset_configs/croatian.py +1 -1
  19. scandeval/dataset_configs/danish.py +3 -2
  20. scandeval/dataset_configs/dutch.py +7 -6
  21. scandeval/dataset_configs/english.py +4 -3
  22. scandeval/dataset_configs/estonian.py +8 -7
  23. scandeval/dataset_configs/faroese.py +1 -1
  24. scandeval/dataset_configs/finnish.py +5 -4
  25. scandeval/dataset_configs/french.py +6 -5
  26. scandeval/dataset_configs/german.py +4 -3
  27. scandeval/dataset_configs/greek.py +1 -1
  28. scandeval/dataset_configs/hungarian.py +1 -1
  29. scandeval/dataset_configs/icelandic.py +4 -3
  30. scandeval/dataset_configs/italian.py +4 -3
  31. scandeval/dataset_configs/latvian.py +2 -2
  32. scandeval/dataset_configs/lithuanian.py +1 -1
  33. scandeval/dataset_configs/norwegian.py +6 -5
  34. scandeval/dataset_configs/polish.py +4 -3
  35. scandeval/dataset_configs/portuguese.py +5 -4
  36. scandeval/dataset_configs/romanian.py +2 -2
  37. scandeval/dataset_configs/serbian.py +1 -1
  38. scandeval/dataset_configs/slovene.py +1 -1
  39. scandeval/dataset_configs/spanish.py +4 -3
  40. scandeval/dataset_configs/swedish.py +4 -3
  41. scandeval/dataset_configs/ukrainian.py +1 -1
  42. scandeval/generation_utils.py +6 -6
  43. scandeval/metrics/llm_as_a_judge.py +1 -1
  44. scandeval/metrics/pipeline.py +1 -1
  45. scandeval/model_cache.py +34 -4
  46. scandeval/prompt_templates/linguistic_acceptability.py +9 -0
  47. scandeval/prompt_templates/multiple_choice.py +9 -0
  48. scandeval/prompt_templates/named_entity_recognition.py +21 -0
  49. scandeval/prompt_templates/reading_comprehension.py +10 -0
  50. scandeval/prompt_templates/sentiment_classification.py +11 -0
  51. scandeval/string_utils.py +157 -0
  52. scandeval/task_group_utils/sequence_classification.py +2 -5
  53. scandeval/task_group_utils/token_classification.py +2 -4
  54. scandeval/utils.py +6 -323
  55. scandeval-16.13.0.dist-info/METADATA +334 -0
  56. scandeval-16.13.0.dist-info/RECORD +94 -0
  57. scandeval-16.12.0.dist-info/METADATA +0 -667
  58. scandeval-16.12.0.dist-info/RECORD +0 -90
  59. {scandeval-16.12.0.dist-info → scandeval-16.13.0.dist-info}/WHEEL +0 -0
  60. {scandeval-16.12.0.dist-info → scandeval-16.13.0.dist-info}/entry_points.txt +0 -0
  61. {scandeval-16.12.0.dist-info → scandeval-16.13.0.dist-info}/licenses/LICENSE +0 -0
scandeval-16.12.0.dist-info/RECORD DELETED
@@ -1,90 +0,0 @@
1
- scandeval/__init__.py,sha256=wHhEEQ8wLNLAN9ULdAkWZpGSo08IpTx_w_gaya0FnVQ,3896
2
- scandeval/benchmark_config_factory.py,sha256=NeikkDCfvTI3ZrAAP-kCQK6Ma3FfwITa_sZ4Ou0w3GM,8895
3
- scandeval/benchmarker.py,sha256=HPG3qF3dX1hnhEc3WYsSGTkWJ8GeXC1ct_A-89IQTtw,54470
4
- scandeval/caching_utils.py,sha256=lLUbkpDdJZy4xodIpwIz5d-WNKGuszbr_d9dyiJ5kZc,2591
5
- scandeval/callbacks.py,sha256=l8f6Zr8EoHfVFsI1ZnMUK0Y8uZB00Nvaz_I6XDn6avE,2515
6
- scandeval/cli.py,sha256=BUrE8ca4wIOQjBM4NoyhNVzGPnVdjOl7xFXbUDuAsq0,9807
7
- scandeval/constants.py,sha256=0IVDd0tmb3r6lKB5CODc4RqS7OofZdW3xE40jT74LeQ,4492
8
- scandeval/data_loading.py,sha256=8ryYEmj6di1f9QefGfNajxObQ9iapIGuAsL8m9KzDyI,7050
9
- scandeval/data_models.py,sha256=IaXgy5OKPA1wHP55-m9IqE2hBC8Kv8nhsUSTqJBq7ho,30968
10
- scandeval/enums.py,sha256=SeFek-Lre2Q5sxbP5svqjDZFZR2vlJhg9dkRH4JvU1g,3436
11
- scandeval/exceptions.py,sha256=4-N2OIo5PJ2aciLjagNAVhdHPxpq2QxywbBqJ8lkKj0,5780
12
- scandeval/finetuning.py,sha256=dTjchPHLFRD65ZrEmtj5TfMTPZ6PODn77t372fgTNwE,11983
13
- scandeval/generation.py,sha256=ccE-S0jxkM99XziIdeaBbk8yRGv4YBkzZkoabhFCSKA,13382
14
- scandeval/generation_utils.py,sha256=A6YCiiMrMEUHq5BcVEjsouIKMPGt0sCfPzsJY1GVyk0,20092
15
- scandeval/languages.py,sha256=gUSosFbvf1eEQHjVsKhXdJ4jiGXC-9lMkOL8AsBG33Q,37295
16
- scandeval/logging_utils.py,sha256=Qnni11ngHrjCf_fgkk6lp6gs-tGSgUS3d5zRR83y6ec,9507
17
- scandeval/model_cache.py,sha256=sjMYW0klnHt2yAFLavDTsp_InxPeSOuVEFo-Rh_31UM,10219
18
- scandeval/model_config.py,sha256=fxHfgpw-9vj3hwke28DguVGvG9TU06nkTXT0V6KAMpQ,2761
19
- scandeval/model_loading.py,sha256=DsX7et18Epcv8kHATZgwPJnwH17GHmh3JCzrSoI3GAE,2377
20
- scandeval/scores.py,sha256=9a1XtppFbp8GJFc9JdThGxqBY0YUE7-92oyrlxScjNk,3281
21
- scandeval/speed_benchmark.py,sha256=VUOvauc9tuAegThNT2g1a-Z1l7DEmKq57dHI4t16o5A,4068
22
- scandeval/tasks.py,sha256=FQvnl28iudjIA2V_G3gHpSsyKaSs7r1i-T5c2pLAuF4,6656
23
- scandeval/tokenisation_utils.py,sha256=K9ovIi5WNqLrFKkafl16R3K-2PallGwV_zeIFw_AM_k,21553
24
- scandeval/types.py,sha256=CHQjLzqKYDXPCyZas7rKg6wD1pNiYuaOFMWimrj5H64,4374
25
- scandeval/utils.py,sha256=P7RARAvJzm-CVavNjMXR2ZseWxT3irXegRzjrVIdCww,17481
26
- scandeval/benchmark_modules/__init__.py,sha256=TNO-sNDwlXE-LMFXfwwqjQqUy55gywSmwRBcoPUFuaU,236
27
- scandeval/benchmark_modules/base.py,sha256=5YAsCMILKTRXFx_ylGQ7iS5AFKN25iFdkBjj8KzzElw,11445
28
- scandeval/benchmark_modules/fresh.py,sha256=sG5ae4p1J-GGmVNcVBIxY1xZIAlUwq_pu-9c4uAYU3Y,10734
29
- scandeval/benchmark_modules/hf.py,sha256=ob-05POUBDWk9dU_hUT7nmXZ11IGCnMgj6xkyLYyX98,48512
30
- scandeval/benchmark_modules/litellm.py,sha256=jVagENE3a0PNMDOaj4DLY-p2Lf-BzNVB1_voPq2CLTU,75545
31
- scandeval/benchmark_modules/vllm.py,sha256=pPKDHf5T_p0u9CJcR7R5sMmN98mirl64kWfyEHbtb5s,61720
32
- scandeval/dataset_configs/__init__.py,sha256=GFI_W9GKd3OSDdhhJzHc8mwoP9b32IHIIyvPBI-hK6k,3223
33
- scandeval/dataset_configs/albanian.py,sha256=D__dli7JO3yeHzzdJ3FFyUGw-z20f1yI6QLnws-WB8I,1473
34
- scandeval/dataset_configs/bosnian.py,sha256=golIWqwW1pFwSkuBM1v0yhHDblB2FoJgK24aO7kKm7M,877
35
- scandeval/dataset_configs/bulgarian.py,sha256=OVoDPTRdU-lVq-xUka7-Ct20h2jbs8HV43KBxRQenIE,1284
36
- scandeval/dataset_configs/catalan.py,sha256=SXwRJjIcMMN7rVuhFRZSnCGDoMfabW5HFoZOkq0Jpg0,1427
37
- scandeval/dataset_configs/croatian.py,sha256=U5oBTjttpWTWonTEzZAf-G3nvQICRQmw6Kla-HWn_5k,1260
38
- scandeval/dataset_configs/czech.py,sha256=ghv2yNw839G-utll8PQRSjyKYbM5gfoQhFKy664GTCI,1562
39
- scandeval/dataset_configs/danish.py,sha256=LEKs04vK2KnV0CYheT7FeS-g3iHBvf2bQxyl0D_LbTg,3293
40
- scandeval/dataset_configs/dutch.py,sha256=q9adDSpR08Ol5AMJJpp1e1T1ZbwmORaFnJaEGrAujm4,3747
41
- scandeval/dataset_configs/english.py,sha256=nc9nGwxf1tHVMUhQeND61yJbpTO4rJaAusPZlstqtq0,2817
42
- scandeval/dataset_configs/estonian.py,sha256=bWiKA_dJ7WUE8Z_1YZnSewhi4ZdCQBGJZ7pQxkCwMcU,2757
43
- scandeval/dataset_configs/faroese.py,sha256=13qYwXonDPWG9Av5MY_NBNTRDglPVKz5_mbz7ZCJ_mo,1247
44
- scandeval/dataset_configs/finnish.py,sha256=eyAMoQE43R-jKD3IZS0iqD2SZOPxWLqjmXzmm00tqPs,2444
45
- scandeval/dataset_configs/french.py,sha256=z6cGY0J7TgXjqmkOLOxQE6ADO1EEPoMF1sdk2n2USe0,2611
46
- scandeval/dataset_configs/german.py,sha256=CXW8_6CMMtrqrq85tDB6gY_fja_FATOqN5glNJI6efU,2858
47
- scandeval/dataset_configs/greek.py,sha256=BLdhfBIG7ABzoZWvtI_VlInpb0SYLK36nhHN33LPVo0,1475
48
- scandeval/dataset_configs/hungarian.py,sha256=1RH7Tt3gleBGuCTDq-Vf87fjTuhjUziY48ZSJHwH0MQ,1421
49
- scandeval/dataset_configs/icelandic.py,sha256=G2Ibe6oF1NknkQmHqLpoHlysW_8f-0G53DJAGAlLkzQ,3552
50
- scandeval/dataset_configs/italian.py,sha256=qhjAQChnQanzs7EyN1DSAJ4OOU41HAlWqWntQOtbWCw,2761
51
- scandeval/dataset_configs/latvian.py,sha256=wbwIDieq5Lplng5Jzx9LEqq4d8b5LnNOyCUmT64b4bA,1928
52
- scandeval/dataset_configs/lithuanian.py,sha256=RPqKwsysO1TYeQuEEsbhzGcSFHDX94lk1hgl1CfQaMU,1724
53
- scandeval/dataset_configs/norwegian.py,sha256=k70T78rTY3pmmVRxG3i_J1j7td_boFHJetkyITskIL0,5487
54
- scandeval/dataset_configs/polish.py,sha256=nN_NT8cUK2iv1L_zO_aCYOk2R7ACSDZgvI7e0hIaFAM,2074
55
- scandeval/dataset_configs/portuguese.py,sha256=m9lEeVtI_yNvIdTIEOn3HFK_ilY2tn3-acC981hjZFM,2401
56
- scandeval/dataset_configs/romanian.py,sha256=AcDp0mqOHmmv3EodovGEcBmarxjLYsXOPr_X4IQoNTw,1472
57
- scandeval/dataset_configs/serbian.py,sha256=yE_aoQRU8qiJqSD-iEt6o_M4rSkGahLX8qz2RRYDvbo,1411
58
- scandeval/dataset_configs/slovak.py,sha256=7JQj2GlRj1giACKqVZ6QJxiFBzP9byoowm0DjPaZ5u8,1252
59
- scandeval/dataset_configs/slovene.py,sha256=r6BbFRvkFYf_4lvQaltaJ1VTVGETZ0xspsu9M_QSLcg,1271
60
- scandeval/dataset_configs/spanish.py,sha256=Q60nx69sGbYk8p0hg2cwLFyoPjg36FdstLQoacw9QmU,2928
61
- scandeval/dataset_configs/swedish.py,sha256=kpEK29swY7iyUSzUvD9hNf2qwb3d7bHrFwboCWVAf2k,3269
62
- scandeval/dataset_configs/ukrainian.py,sha256=spbCmCOU27jOfz6FZxqCIfVmDN5l8H-7VCl-k-8eAIo,1527
63
- scandeval/metrics/__init__.py,sha256=nrjFjTK7NO5I8U6acULNzqezmMWN21aWd4faW4oYGHo,233
64
- scandeval/metrics/base.py,sha256=dUBby-ZzettMjdcjek6rw0JTZMuScX4cQ2Rd6untKHY,2525
65
- scandeval/metrics/bias.py,sha256=sV87PLzjc3XPsSAz2HJ4hmlLZ_IcHDsIUr7gYmp9HKc,7765
66
- scandeval/metrics/huggingface.py,sha256=eKXn5wBcNdzs23cgJ64XG8LIwen1wDxXy2kAOw3bjoQ,9579
67
- scandeval/metrics/llm_as_a_judge.py,sha256=UUFk3aL2BZqJ-u9-dzexsoArTxPJTMmHRqb1eWxexaI,12133
68
- scandeval/metrics/pipeline.py,sha256=GTIqaFkn-nTLU4xBi8-zP1J4Ytv3qeFVuRB4OcuwkOw,10876
69
- scandeval/metrics/speed.py,sha256=G5hEQcrtqxF070ZZwLDh61iZnq2CSW2o6ZM7zR4lOTY,1298
70
- scandeval/prompt_templates/__init__.py,sha256=p3CUcSaJiiUm6EQyhceDUjotH7GdyHolMznAn2f44as,519
71
- scandeval/prompt_templates/classification.py,sha256=QuZh6hTMaqMYTsoruAhwjVP9381zzlQmDIwSeyGnav0,10121
72
- scandeval/prompt_templates/linguistic_acceptability.py,sha256=V31apMLPNhTeDJO6va_04SjuDSXMOJEFurIeSldDi7o,15474
73
- scandeval/prompt_templates/multiple_choice.py,sha256=pgz-Xb-vUthwJyjla56CxeeXPDkgtZ7Mi9z1J-PjepY,12977
74
- scandeval/prompt_templates/named_entity_recognition.py,sha256=U9KYr4eIbiMdHECc35CjkNUDoiRd6Jd8w0v35kRWGL4,30197
75
- scandeval/prompt_templates/reading_comprehension.py,sha256=4C16Mf1MGtEZG9x8PxrJmK1Cxfz9kzjrJLNS725_5oI,16319
76
- scandeval/prompt_templates/sentiment_classification.py,sha256=mLrhWh0rQTjiowzprv8S5CfLO_g7DvnSjWiw0CsaXpg,17401
77
- scandeval/prompt_templates/simplification.py,sha256=DF50F1JSxy00ZOO3OJJZOtoTlkGjE35krjjbDaW7RUk,900
78
- scandeval/prompt_templates/summarization.py,sha256=LKiz5fd6A0J5NyoLBeyrZ4ir1skDB2pytKCEeF4zbmw,10770
79
- scandeval/prompt_templates/token_classification.py,sha256=8Uw34mN2xQ_5es-nz7vCK-GgDg_oE-zsAzPJPzAxFrQ,15531
80
- scandeval/task_group_utils/__init__.py,sha256=CorGVkixkoEDOQuDsrOGlTmF1zmM0wnGHs8psWTfD28,72
81
- scandeval/task_group_utils/multiple_choice_classification.py,sha256=PWUXeGn-9RsXxdVRYHJASyBVQ8L5Jla981eot0GLooY,7316
82
- scandeval/task_group_utils/question_answering.py,sha256=tuMwr-RnvJap5jkTrluxC1tfQVS6rKN8_ifNwis-auw,29064
83
- scandeval/task_group_utils/sequence_classification.py,sha256=1YAaKn5bY8j9ONPfJZODjaGKVMkA9fQcl51fvBcjeF8,16829
84
- scandeval/task_group_utils/text_to_text.py,sha256=p6zzjob70qQUpfUOs0LToSzavE1ERqRAHu_727Jb2mM,5476
85
- scandeval/task_group_utils/token_classification.py,sha256=8dF32KQAYAFnnn7DPHX-yvJmRrMBmT2CyFREacyTwvQ,17321
86
- scandeval-16.12.0.dist-info/METADATA,sha256=YCSgBbbtWLDfWqepHFS8UX0zho8gpTXJC1lagT_l94w,24564
87
- scandeval-16.12.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
88
- scandeval-16.12.0.dist-info/entry_points.txt,sha256=-mtBu-10bFWeZ2bS32gVK6-s-LNCQLxvnNUPBLd5ud4,87
89
- scandeval-16.12.0.dist-info/licenses/LICENSE,sha256=vb2c84xITVnhnVFsBS8AWXl-4S-KpxN6VMxTqqYlV3s,1080
90
- scandeval-16.12.0.dist-info/RECORD,,