evalscope 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of evalscope might be problematic. Click here for more details.

Files changed (178) hide show
  1. evalscope/arguments.py +2 -1
  2. evalscope/benchmarks/__init__.py +2 -2
  3. evalscope/benchmarks/aigc/__init__.py +0 -0
  4. evalscope/benchmarks/aigc/t2i/__init__.py +0 -0
  5. evalscope/benchmarks/aigc/t2i/base.py +56 -0
  6. evalscope/benchmarks/aigc/t2i/evalmuse_adapter.py +77 -0
  7. evalscope/benchmarks/aigc/t2i/genai_bench_adapter.py +58 -0
  8. evalscope/benchmarks/aigc/t2i/general_t2i_adapter.py +58 -0
  9. evalscope/benchmarks/aigc/t2i/hpdv2_adapter.py +57 -0
  10. evalscope/benchmarks/aigc/t2i/tifa_adapter.py +37 -0
  11. evalscope/benchmarks/aime/aime24_adapter.py +1 -1
  12. evalscope/benchmarks/aime/aime25_adapter.py +4 -4
  13. evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py +1 -2
  14. evalscope/benchmarks/arc/arc_adapter.py +1 -1
  15. evalscope/benchmarks/arena_hard/arena_hard_adapter.py +1 -3
  16. evalscope/benchmarks/ceval/ceval_adapter.py +2 -2
  17. evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py +1 -3
  18. evalscope/benchmarks/cmmlu/cmmlu_adapter.py +1 -1
  19. evalscope/benchmarks/competition_math/competition_math_adapter.py +1 -2
  20. evalscope/benchmarks/data_adapter.py +16 -9
  21. evalscope/benchmarks/data_collection/data_collection_adapter.py +6 -4
  22. evalscope/benchmarks/general_mcq/general_mcq_adapter.py +2 -2
  23. evalscope/benchmarks/live_code_bench/evaluate_utils.py +16 -21
  24. evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py +4 -1
  25. evalscope/benchmarks/live_code_bench/testing_util.py +6 -3
  26. evalscope/benchmarks/math_500/math_500_adapter.py +1 -1
  27. evalscope/benchmarks/mmlu/mmlu_adapter.py +3 -1
  28. evalscope/benchmarks/simple_qa/simple_qa_adapter.py +1 -2
  29. evalscope/benchmarks/utils.py +7 -16
  30. evalscope/cli/start_app.py +1 -1
  31. evalscope/collections/evaluator.py +16 -4
  32. evalscope/config.py +7 -3
  33. evalscope/constants.py +11 -0
  34. evalscope/evaluator/evaluator.py +2 -2
  35. evalscope/evaluator/reviewer/auto_reviewer.py +1 -1
  36. evalscope/metrics/__init__.py +49 -4
  37. evalscope/metrics/llm_judge.py +1 -1
  38. evalscope/metrics/named_metrics.py +13 -0
  39. evalscope/metrics/t2v_metrics/__init__.py +66 -0
  40. evalscope/metrics/t2v_metrics/clipscore.py +14 -0
  41. evalscope/metrics/t2v_metrics/constants.py +12 -0
  42. evalscope/metrics/t2v_metrics/itmscore.py +14 -0
  43. evalscope/metrics/t2v_metrics/models/__init__.py +0 -0
  44. evalscope/metrics/t2v_metrics/models/clipscore_models/__init__.py +30 -0
  45. evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/__init__.py +0 -0
  46. evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/base_model.py +6 -0
  47. evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/clip_model.py +132 -0
  48. evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/cross_modeling.py +286 -0
  49. evalscope/metrics/t2v_metrics/models/clipscore_models/clip_model.py +114 -0
  50. evalscope/metrics/t2v_metrics/models/clipscore_models/hpsv2_model.py +86 -0
  51. evalscope/metrics/t2v_metrics/models/clipscore_models/mps_model.py +85 -0
  52. evalscope/metrics/t2v_metrics/models/clipscore_models/pickscore_model.py +62 -0
  53. evalscope/metrics/t2v_metrics/models/itmscore_models/__init__.py +26 -0
  54. evalscope/metrics/t2v_metrics/models/itmscore_models/blip2_itm_model.py +84 -0
  55. evalscope/metrics/t2v_metrics/models/itmscore_models/fga_blip2_model.py +97 -0
  56. evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward/ImageReward.py +171 -0
  57. evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward/__init__.py +0 -0
  58. evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward/blip_pretrain.py +80 -0
  59. evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward_model.py +73 -0
  60. evalscope/metrics/t2v_metrics/models/model.py +45 -0
  61. evalscope/metrics/t2v_metrics/models/utils.py +25 -0
  62. evalscope/metrics/t2v_metrics/models/vqascore_models/__init__.py +22 -0
  63. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/__init__.py +0 -0
  64. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/__init__.py +1 -0
  65. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/language_model/clip_t5.py +300 -0
  66. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/multimodal_encoder/builder.py +12 -0
  67. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/multimodal_encoder/clip_encoder.py +82 -0
  68. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/multimodal_projector/builder.py +50 -0
  69. evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5_model.py +218 -0
  70. evalscope/metrics/t2v_metrics/models/vqascore_models/gpt4v_model.py +150 -0
  71. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/__init__.py +26 -0
  72. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/config.py +465 -0
  73. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/dist_utils.py +141 -0
  74. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/gradcam.py +22 -0
  75. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/logger.py +188 -0
  76. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/optims.py +106 -0
  77. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/registry.py +307 -0
  78. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/utils.py +416 -0
  79. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/vqa_tools/__init__.py +8 -0
  80. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/vqa_tools/vqa.py +191 -0
  81. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/vqa_tools/vqa_eval.py +318 -0
  82. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/default.yaml +10 -0
  83. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_caption_flant5xl.yaml +42 -0
  84. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_caption_opt2.7b.yaml +42 -0
  85. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_caption_opt6.7b.yaml +42 -0
  86. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_coco.yaml +36 -0
  87. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_instruct_flant5xl.yaml +43 -0
  88. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_instruct_flant5xxl.yaml +43 -0
  89. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_instruct_vicuna13b.yaml +43 -0
  90. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml +43 -0
  91. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain.yaml +36 -0
  92. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_flant5xl.yaml +42 -0
  93. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_flant5xl_iter_80k_total_100k_no_prefix.yaml +42 -0
  94. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_flant5xl_iter_80k_total_100k_prefix.yaml +42 -0
  95. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_flant5xl_vitL.yaml +43 -0
  96. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_flant5xxl.yaml +42 -0
  97. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_opt2.7b.yaml +42 -0
  98. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_opt6.7b.yaml +42 -0
  99. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_pretrain_vitL.yaml +37 -0
  100. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_vicuna13b.yaml +43 -0
  101. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/blip2/blip2_vicuna7b.yaml +43 -0
  102. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/med_config.json +21 -0
  103. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/med_config_albef.json +22 -0
  104. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/configs/models/med_large_config.json +21 -0
  105. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/__init__.py +208 -0
  106. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/base_model.py +231 -0
  107. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/Qformer.py +1093 -0
  108. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/__init__.py +0 -0
  109. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/blip2.py +211 -0
  110. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/blip2_image_text_matching.py +109 -0
  111. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/blip2_qformer.py +452 -0
  112. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/blip2_t5.py +364 -0
  113. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/blip2_t5_instruct.py +755 -0
  114. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/fga_blip2.py +273 -0
  115. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/modeling_llama.py +880 -0
  116. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/modeling_t5.py +1844 -0
  117. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/__init__.py +81 -0
  118. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip.py +56 -0
  119. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_caption.py +212 -0
  120. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_classification.py +164 -0
  121. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_feature_extractor.py +202 -0
  122. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_image_text_matching.py +185 -0
  123. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_nlvr.py +178 -0
  124. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_outputs.py +112 -0
  125. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_pretrain.py +371 -0
  126. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_vqa.py +344 -0
  127. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/nlvr_encoder.py +858 -0
  128. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/clip_vit.py +271 -0
  129. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/eva_vit.py +503 -0
  130. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/med.py +1270 -0
  131. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/vit.py +473 -0
  132. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/__init__.py +31 -0
  133. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/base_processor.py +27 -0
  134. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/blip_processors.py +233 -0
  135. evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/randaugment.py +392 -0
  136. evalscope/metrics/t2v_metrics/models/vqascore_models/mm_utils.py +127 -0
  137. evalscope/metrics/t2v_metrics/models/vqascore_models/vqa_model.py +17 -0
  138. evalscope/metrics/t2v_metrics/score.py +78 -0
  139. evalscope/metrics/t2v_metrics/vqascore.py +14 -0
  140. evalscope/models/__init__.py +50 -14
  141. evalscope/models/adapters/__init__.py +17 -0
  142. evalscope/models/{base_adapter.py → adapters/base_adapter.py} +17 -17
  143. evalscope/models/{chat_adapter.py → adapters/chat_adapter.py} +10 -7
  144. evalscope/models/{choice_adapter.py → adapters/choice_adapter.py} +2 -6
  145. evalscope/models/{custom_adapter.py → adapters/custom_adapter.py} +2 -4
  146. evalscope/models/{server_adapter.py → adapters/server_adapter.py} +1 -3
  147. evalscope/models/adapters/t2i_adapter.py +76 -0
  148. evalscope/models/custom/__init__.py +2 -1
  149. evalscope/models/custom/dummy_model.py +11 -13
  150. evalscope/models/local_model.py +82 -33
  151. evalscope/models/model.py +2 -42
  152. evalscope/models/register.py +26 -0
  153. evalscope/perf/plugin/datasets/flickr8k.py +2 -1
  154. evalscope/perf/utils/benchmark_util.py +2 -2
  155. evalscope/perf/utils/db_util.py +8 -2
  156. evalscope/report/__init__.py +1 -0
  157. evalscope/report/app.py +117 -67
  158. evalscope/report/app_arguments.py +11 -0
  159. evalscope/report/generator.py +1 -1
  160. evalscope/run.py +3 -3
  161. evalscope/third_party/thinkbench/eval.py +19 -7
  162. evalscope/utils/chat_service.py +2 -2
  163. evalscope/utils/import_utils.py +66 -0
  164. evalscope/utils/utils.py +12 -4
  165. evalscope/version.py +2 -2
  166. {evalscope-0.14.0.dist-info → evalscope-0.15.0.dist-info}/METADATA +18 -1
  167. {evalscope-0.14.0.dist-info → evalscope-0.15.0.dist-info}/RECORD +175 -63
  168. tests/aigc/__init__.py +1 -0
  169. tests/aigc/test_t2i.py +87 -0
  170. tests/cli/test_run.py +11 -5
  171. tests/perf/test_perf.py +2 -1
  172. evalscope/metrics/code_metric.py +0 -98
  173. evalscope/metrics/resources/gpt2-zhcn3-v4.bpe +0 -58485
  174. evalscope/metrics/resources/gpt2-zhcn3-v4.json +0 -1
  175. {evalscope-0.14.0.dist-info → evalscope-0.15.0.dist-info}/LICENSE +0 -0
  176. {evalscope-0.14.0.dist-info → evalscope-0.15.0.dist-info}/WHEEL +0 -0
  177. {evalscope-0.14.0.dist-info → evalscope-0.15.0.dist-info}/entry_points.txt +0 -0
  178. {evalscope-0.14.0.dist-info → evalscope-0.15.0.dist-info}/top_level.txt +0 -0

There are too many changes on this page to be displayed.

The amount of changes on this page would crash your brower.

You can still verify the content by downloading the package file manually.