evalscope 0.5.2__py3-none-any.whl → 0.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of evalscope might be problematic. Click here for more details.
- evalscope/backend/opencompass/tasks/eval_datasets.py +1 -0
- evalscope/backend/vlm_eval_kit/backend_manager.py +11 -6
- evalscope/backend/vlm_eval_kit/custom_dataset.py +47 -0
- evalscope/config.py +1 -0
- evalscope/models/model_adapter.py +6 -0
- evalscope/run_arena.py +5 -3
- evalscope/summarizer.py +10 -4
- evalscope/version.py +2 -2
- evalscope-0.5.3.dist-info/METADATA +407 -0
- {evalscope-0.5.2.dist-info → evalscope-0.5.3.dist-info}/RECORD +13 -12
- evalscope-0.5.2.dist-info/METADATA +0 -578
- {evalscope-0.5.2.dist-info → evalscope-0.5.3.dist-info}/WHEEL +0 -0
- {evalscope-0.5.2.dist-info → evalscope-0.5.3.dist-info}/entry_points.txt +0 -0
- {evalscope-0.5.2.dist-info → evalscope-0.5.3.dist-info}/top_level.txt +0 -0
|
@@ -49,6 +49,7 @@ with read_base():
|
|
|
49
49
|
from opencompass.configs.datasets.obqa.obqa_gen_9069e4 import obqa_datasets
|
|
50
50
|
from opencompass.configs.datasets.nq.nq_gen_c788f6 import nq_datasets
|
|
51
51
|
from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
|
|
52
|
+
from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
|
|
52
53
|
from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
|
|
53
54
|
|
|
54
55
|
# Note: to be supported
|
|
@@ -37,6 +37,7 @@ class VLMEvalKitBackendManager(BackendManager):
|
|
|
37
37
|
|
|
38
38
|
self._check_valid()
|
|
39
39
|
|
|
40
|
+
|
|
40
41
|
def _check_valid(self):
|
|
41
42
|
# Ensure not both model and datasets are empty
|
|
42
43
|
if not self.args.data or not self.args.model:
|
|
@@ -44,9 +45,9 @@ class VLMEvalKitBackendManager(BackendManager):
|
|
|
44
45
|
|
|
45
46
|
# Check datasets
|
|
46
47
|
valid_datasets, invalid_datasets = get_valid_list(self.args.data, self.valid_datasets)
|
|
47
|
-
|
|
48
|
-
f
|
|
49
|
-
|
|
48
|
+
if len(invalid_datasets) != 0:
|
|
49
|
+
logger.warning(f"Using custom dataset: {invalid_datasets}, ")
|
|
50
|
+
|
|
50
51
|
# Check model
|
|
51
52
|
if isinstance(self.args.model[0], dict):
|
|
52
53
|
model_names = [model['name'] for model in self.args.model]
|
|
@@ -61,10 +62,14 @@ class VLMEvalKitBackendManager(BackendManager):
|
|
|
61
62
|
model_class = self.valid_models[model_name]
|
|
62
63
|
if model_name == 'CustomAPIModel':
|
|
63
64
|
model_type = model_cfg['type']
|
|
65
|
+
remain_cfg = copy.deepcopy(model_cfg)
|
|
66
|
+
del remain_cfg['name'] # remove not used args
|
|
67
|
+
del remain_cfg['type'] # remove not used args
|
|
68
|
+
|
|
64
69
|
self.valid_models.update({
|
|
65
70
|
model_type: partial(model_class,
|
|
66
71
|
model=model_type,
|
|
67
|
-
**
|
|
72
|
+
**remain_cfg)
|
|
68
73
|
})
|
|
69
74
|
new_model_names.append(model_type)
|
|
70
75
|
else:
|
|
@@ -78,8 +83,8 @@ class VLMEvalKitBackendManager(BackendManager):
|
|
|
78
83
|
|
|
79
84
|
elif isinstance(self.args.model[0], str):
|
|
80
85
|
valid_model_names, invalid_model_names = get_valid_list(self.args.model, self.valid_model_names)
|
|
81
|
-
|
|
82
|
-
f
|
|
86
|
+
if len(invalid_datasets) != 0:
|
|
87
|
+
logger.warning(f"Using custom dataset: {invalid_datasets}, ")
|
|
83
88
|
|
|
84
89
|
@property
|
|
85
90
|
def cmd(self):
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import numpy as np
|
|
3
|
+
from vlmeval.dataset.image_base import ImageBaseDataset
|
|
4
|
+
from vlmeval.dataset.image_vqa import CustomVQADataset
|
|
5
|
+
from vlmeval.smp import load, dump, d2df
|
|
6
|
+
|
|
7
|
+
class CustomDataset:
|
|
8
|
+
|
|
9
|
+
def load_data(self, dataset):
|
|
10
|
+
# customize the loading of the dataset
|
|
11
|
+
data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
|
|
12
|
+
return load(data_path)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def build_prompt(self, line):
|
|
16
|
+
msgs = ImageBaseDataset.build_prompt(self, line)
|
|
17
|
+
# add a hint or custom instruction here
|
|
18
|
+
msgs[-1]['value'] += '\nAnswer the question using a single word or phrase.'
|
|
19
|
+
return msgs
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def evaluate(self, eval_file, **judge_kwargs):
|
|
23
|
+
data = load(eval_file)
|
|
24
|
+
assert 'answer' in data and 'prediction' in data
|
|
25
|
+
data['prediction'] = [str(x) for x in data['prediction']]
|
|
26
|
+
data['answer'] = [str(x).lower() for x in data['answer']]
|
|
27
|
+
|
|
28
|
+
print(data)
|
|
29
|
+
|
|
30
|
+
# ========compute the evaluation metrics as you need =========
|
|
31
|
+
# exact match
|
|
32
|
+
result = np.mean(data['answer'] == data['prediction'])
|
|
33
|
+
ret = {'Overall': result}
|
|
34
|
+
ret = d2df(ret).round(2)
|
|
35
|
+
|
|
36
|
+
# save the result
|
|
37
|
+
suffix = eval_file.split('.')[-1]
|
|
38
|
+
result_file = eval_file.replace(f'.{suffix}', '_acc.csv')
|
|
39
|
+
dump(ret, result_file)
|
|
40
|
+
return ret
|
|
41
|
+
# ============================================================
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# override the default dataset class
|
|
45
|
+
CustomVQADataset.load_data = CustomDataset.load_data
|
|
46
|
+
CustomVQADataset.build_prompt = CustomDataset.build_prompt
|
|
47
|
+
CustomVQADataset.evaluate = CustomDataset.evaluate
|
evalscope/config.py
CHANGED
|
@@ -33,6 +33,7 @@ registry_tasks = {
|
|
|
33
33
|
@dataclass
|
|
34
34
|
class TaskConfig:
|
|
35
35
|
model_args: Optional[dict] = field(default_factory=dict)
|
|
36
|
+
template_type: Optional[str] = 'default-generation'
|
|
36
37
|
generation_config: Optional[dict] = field(default_factory=dict)
|
|
37
38
|
dataset_args: Optional[dict] = field(default_factory=dict)
|
|
38
39
|
dry_run: bool = False
|
|
@@ -362,6 +362,8 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
|
|
|
362
362
|
torch_dtype: The torch dtype for model inference. Default: torch.float16.
|
|
363
363
|
**kwargs: Other args.
|
|
364
364
|
"""
|
|
365
|
+
|
|
366
|
+
custom_generation_config = kwargs.pop('generation_config', None)
|
|
365
367
|
model_cache_dir = get_model_cache_dir(root_cache_dir=cache_dir)
|
|
366
368
|
|
|
367
369
|
self.model_id: str = model_id
|
|
@@ -414,6 +416,10 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
|
|
|
414
416
|
self.origin_tokenizer = deepcopy(tokenizer)
|
|
415
417
|
|
|
416
418
|
self.generation_config, self.generation_template = self._parse_generation_config(tokenizer, model)
|
|
419
|
+
|
|
420
|
+
if custom_generation_config:
|
|
421
|
+
logger.info('**Updating generation config ...')
|
|
422
|
+
self.generation_config.update(**custom_generation_config.to_dict())
|
|
417
423
|
logger.info(f'**Generation config init: {self.generation_config.to_dict()}')
|
|
418
424
|
|
|
419
425
|
super().__init__(model=model, tokenizer=self.generation_template.tokenizer, model_cfg=model_cfg)
|
evalscope/run_arena.py
CHANGED
|
@@ -100,17 +100,18 @@ class ArenaWorkflow:
|
|
|
100
100
|
model_revision = cfg_d.get(EvalConfigKeys.MODEL_REVISION, None)
|
|
101
101
|
precision = cfg_d.get(EvalConfigKeys.PRECISION, torch.float16)
|
|
102
102
|
precision = eval(precision) if isinstance(precision, str) else precision
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
custom_generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
|
|
104
|
+
custom_generation_config = GenerationConfig(**custom_generation_config)
|
|
105
105
|
ans_output_file = os.path.join(WORK_DIR, cfg_d.get(EvalConfigKeys.OUTPUT_FILE))
|
|
106
106
|
template_type = cfg_d.get(EvalConfigKeys.TEMPLATE_TYPE)
|
|
107
107
|
|
|
108
108
|
answers_list = self._predict_answers(model_id_or_path=model_id_or_path,
|
|
109
109
|
model_revision=model_revision,
|
|
110
110
|
precision=precision,
|
|
111
|
-
generation_config=
|
|
111
|
+
generation_config=custom_generation_config,
|
|
112
112
|
template_type=template_type)
|
|
113
113
|
|
|
114
|
+
os.makedirs(os.path.dirname(ans_output_file), exist_ok=True)
|
|
114
115
|
dump_jsonl_data(answers_list, ans_output_file)
|
|
115
116
|
logger.info(f'Answers generated by model {model_name} and saved to {ans_output_file}')
|
|
116
117
|
|
|
@@ -168,6 +169,7 @@ class ArenaWorkflow:
|
|
|
168
169
|
res_list = ae.run(self.review_file)
|
|
169
170
|
rating_df = res_list[0]
|
|
170
171
|
logger.info(f'Rating results:\n{rating_df.to_csv()}')
|
|
172
|
+
os.makedirs(os.path.dirname(report_file), exist_ok=True)
|
|
171
173
|
rating_df.to_csv(report_file, index=True)
|
|
172
174
|
logger.info(f'Rating results are saved to {report_file}')
|
|
173
175
|
else:
|
evalscope/summarizer.py
CHANGED
|
@@ -99,19 +99,25 @@ class Summarizer:
|
|
|
99
99
|
elif eval_backend == EvalBackend.VLM_EVAL_KIT.value:
|
|
100
100
|
eval_config = Summarizer.parse_eval_config(candidate_task)
|
|
101
101
|
|
|
102
|
-
work_dir = eval_config.get('work_dir') or 'outputs
|
|
102
|
+
work_dir = eval_config.get('work_dir') or 'outputs'
|
|
103
103
|
if not os.path.exists(work_dir):
|
|
104
104
|
raise ValueError(f'work_dir {work_dir} does not exist.')
|
|
105
105
|
|
|
106
|
-
# TODO: parse summary files: acc.csv, score.csv, score.json for different models
|
|
107
106
|
for model in eval_config['model']:
|
|
108
107
|
if model['name'] == 'CustomAPIModel':
|
|
109
108
|
model_name = model['type']
|
|
110
109
|
else:
|
|
111
110
|
model_name = model['name']
|
|
112
|
-
|
|
111
|
+
|
|
112
|
+
csv_files = glob.glob(os.path.join(work_dir, model_name, '*.csv'))
|
|
113
|
+
json_files = glob.glob(os.path.join(work_dir, model_name, '*.json'))
|
|
114
|
+
|
|
115
|
+
summary_files = csv_files + json_files
|
|
113
116
|
for summary_file_path in summary_files:
|
|
114
|
-
|
|
117
|
+
if summary_file_path.endswith('csv'):
|
|
118
|
+
summary_res: dict = csv_to_list(summary_file_path)[0]
|
|
119
|
+
elif summary_file_path.endswith('json'):
|
|
120
|
+
summary_res: dict = json_to_dict(summary_file_path)
|
|
115
121
|
file_name = os.path.basename(summary_file_path).split('.')[0]
|
|
116
122
|
final_res_list.append({file_name: summary_res})
|
|
117
123
|
|
evalscope/version.py
CHANGED
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: evalscope
|
|
3
|
+
Version: 0.5.3
|
|
4
|
+
Summary: EvalScope: Lightweight LLMs Evaluation Framework
|
|
5
|
+
Home-page: https://github.com/modelscope/evalscope
|
|
6
|
+
Author: ModelScope team
|
|
7
|
+
Author-email: contact@modelscope.cn
|
|
8
|
+
Keywords: python,llm,evaluation
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: torch
|
|
19
|
+
Requires-Dist: absl-py
|
|
20
|
+
Requires-Dist: accelerate
|
|
21
|
+
Requires-Dist: cachetools
|
|
22
|
+
Requires-Dist: editdistance
|
|
23
|
+
Requires-Dist: jsonlines
|
|
24
|
+
Requires-Dist: matplotlib
|
|
25
|
+
Requires-Dist: modelscope[framework]
|
|
26
|
+
Requires-Dist: nltk
|
|
27
|
+
Requires-Dist: openai
|
|
28
|
+
Requires-Dist: pandas
|
|
29
|
+
Requires-Dist: plotly
|
|
30
|
+
Requires-Dist: pyarrow
|
|
31
|
+
Requires-Dist: pympler
|
|
32
|
+
Requires-Dist: pyyaml
|
|
33
|
+
Requires-Dist: regex
|
|
34
|
+
Requires-Dist: requests
|
|
35
|
+
Requires-Dist: requests-toolbelt
|
|
36
|
+
Requires-Dist: rouge-score
|
|
37
|
+
Requires-Dist: sacrebleu
|
|
38
|
+
Requires-Dist: scikit-learn
|
|
39
|
+
Requires-Dist: seaborn
|
|
40
|
+
Requires-Dist: sentencepiece
|
|
41
|
+
Requires-Dist: simple-ddl-parser
|
|
42
|
+
Requires-Dist: tabulate
|
|
43
|
+
Requires-Dist: tiktoken
|
|
44
|
+
Requires-Dist: tqdm
|
|
45
|
+
Requires-Dist: transformers (<4.43,>=4.33)
|
|
46
|
+
Requires-Dist: transformers-stream-generator
|
|
47
|
+
Requires-Dist: jieba
|
|
48
|
+
Requires-Dist: rouge-chinese
|
|
49
|
+
Provides-Extra: all
|
|
50
|
+
Requires-Dist: torch ; extra == 'all'
|
|
51
|
+
Requires-Dist: absl-py ; extra == 'all'
|
|
52
|
+
Requires-Dist: accelerate ; extra == 'all'
|
|
53
|
+
Requires-Dist: cachetools ; extra == 'all'
|
|
54
|
+
Requires-Dist: editdistance ; extra == 'all'
|
|
55
|
+
Requires-Dist: jsonlines ; extra == 'all'
|
|
56
|
+
Requires-Dist: matplotlib ; extra == 'all'
|
|
57
|
+
Requires-Dist: modelscope[framework] ; extra == 'all'
|
|
58
|
+
Requires-Dist: nltk ; extra == 'all'
|
|
59
|
+
Requires-Dist: openai ; extra == 'all'
|
|
60
|
+
Requires-Dist: pandas ; extra == 'all'
|
|
61
|
+
Requires-Dist: plotly ; extra == 'all'
|
|
62
|
+
Requires-Dist: pyarrow ; extra == 'all'
|
|
63
|
+
Requires-Dist: pympler ; extra == 'all'
|
|
64
|
+
Requires-Dist: pyyaml ; extra == 'all'
|
|
65
|
+
Requires-Dist: regex ; extra == 'all'
|
|
66
|
+
Requires-Dist: requests ; extra == 'all'
|
|
67
|
+
Requires-Dist: requests-toolbelt ; extra == 'all'
|
|
68
|
+
Requires-Dist: rouge-score ; extra == 'all'
|
|
69
|
+
Requires-Dist: sacrebleu ; extra == 'all'
|
|
70
|
+
Requires-Dist: scikit-learn ; extra == 'all'
|
|
71
|
+
Requires-Dist: seaborn ; extra == 'all'
|
|
72
|
+
Requires-Dist: sentencepiece ; extra == 'all'
|
|
73
|
+
Requires-Dist: simple-ddl-parser ; extra == 'all'
|
|
74
|
+
Requires-Dist: tabulate ; extra == 'all'
|
|
75
|
+
Requires-Dist: tiktoken ; extra == 'all'
|
|
76
|
+
Requires-Dist: tqdm ; extra == 'all'
|
|
77
|
+
Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'all'
|
|
78
|
+
Requires-Dist: transformers-stream-generator ; extra == 'all'
|
|
79
|
+
Requires-Dist: jieba ; extra == 'all'
|
|
80
|
+
Requires-Dist: rouge-chinese ; extra == 'all'
|
|
81
|
+
Requires-Dist: ms-opencompass (>=0.1.0) ; extra == 'all'
|
|
82
|
+
Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'all'
|
|
83
|
+
Provides-Extra: inner
|
|
84
|
+
Requires-Dist: absl-py ; extra == 'inner'
|
|
85
|
+
Requires-Dist: accelerate ; extra == 'inner'
|
|
86
|
+
Requires-Dist: alibaba-itag-sdk ; extra == 'inner'
|
|
87
|
+
Requires-Dist: dashscope ; extra == 'inner'
|
|
88
|
+
Requires-Dist: editdistance ; extra == 'inner'
|
|
89
|
+
Requires-Dist: jsonlines ; extra == 'inner'
|
|
90
|
+
Requires-Dist: nltk ; extra == 'inner'
|
|
91
|
+
Requires-Dist: openai ; extra == 'inner'
|
|
92
|
+
Requires-Dist: pandas (==1.5.3) ; extra == 'inner'
|
|
93
|
+
Requires-Dist: plotly ; extra == 'inner'
|
|
94
|
+
Requires-Dist: pyarrow ; extra == 'inner'
|
|
95
|
+
Requires-Dist: pyodps ; extra == 'inner'
|
|
96
|
+
Requires-Dist: pyyaml ; extra == 'inner'
|
|
97
|
+
Requires-Dist: regex ; extra == 'inner'
|
|
98
|
+
Requires-Dist: requests (==2.28.1) ; extra == 'inner'
|
|
99
|
+
Requires-Dist: requests-toolbelt (==0.10.1) ; extra == 'inner'
|
|
100
|
+
Requires-Dist: rouge-score ; extra == 'inner'
|
|
101
|
+
Requires-Dist: sacrebleu ; extra == 'inner'
|
|
102
|
+
Requires-Dist: scikit-learn ; extra == 'inner'
|
|
103
|
+
Requires-Dist: seaborn ; extra == 'inner'
|
|
104
|
+
Requires-Dist: simple-ddl-parser ; extra == 'inner'
|
|
105
|
+
Requires-Dist: streamlit ; extra == 'inner'
|
|
106
|
+
Requires-Dist: tqdm ; extra == 'inner'
|
|
107
|
+
Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'inner'
|
|
108
|
+
Requires-Dist: transformers-stream-generator ; extra == 'inner'
|
|
109
|
+
Provides-Extra: opencompass
|
|
110
|
+
Requires-Dist: ms-opencompass (>=0.1.0) ; extra == 'opencompass'
|
|
111
|
+
Provides-Extra: vlmeval
|
|
112
|
+
Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'vlmeval'
|
|
113
|
+
|
|
114
|
+
English | [简体中文](README_zh.md)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
<p align="center">
|
|
120
|
+
<a href="https://badge.fury.io/py/evalscope"><img src="https://badge.fury.io/py/evalscope.svg" alt="PyPI version" height="18"></a>
|
|
121
|
+
<a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://static.pepy.tech/badge/evalscope">
|
|
122
|
+
</a>
|
|
123
|
+
<a href='https://evalscope.readthedocs.io/en/latest/?badge=latest'>
|
|
124
|
+
<img src='https://readthedocs.org/projects/evalscope-en/badge/?version=latest' alt='Documentation Status' />
|
|
125
|
+
</a>
|
|
126
|
+
<br>
|
|
127
|
+
<a href="https://evalscope.readthedocs.io/en/latest/"><span style="font-size: 16px;">📖 Documents</span></a>   |  <a href="https://evalscope.readthedocs.io/zh-cn/latest/"><span style="font-size: 16px;"> 📖 中文文档</span></a>
|
|
128
|
+
<p>
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
## 📋 Table of Contents
|
|
132
|
+
- [Introduction](#introduction)
|
|
133
|
+
- [News](#News)
|
|
134
|
+
- [Installation](#installation)
|
|
135
|
+
- [Quick Start](#quick-start)
|
|
136
|
+
- [Evaluation Backend](#evaluation-backend)
|
|
137
|
+
- [Custom Dataset Evaluation](#custom-dataset-evaluation)
|
|
138
|
+
- [Offline Evaluation](#offline-evaluation)
|
|
139
|
+
- [Arena Mode](#arena-mode)
|
|
140
|
+
- [Model Serving Performance Evaluation](#Model-Serving-Performance-Evaluation)
|
|
141
|
+
- [Leaderboard](#leaderboard)
|
|
142
|
+
|
|
143
|
+
## 📝 Introduction
|
|
144
|
+
|
|
145
|
+
Large Model (including Large Language Models, Multi-modal Large Language Models) evaluation has become a critical process for assessing and improving LLMs. To better support the evaluation of large models, we propose the EvalScope framework.
|
|
146
|
+
|
|
147
|
+
### Framework Features
|
|
148
|
+
- **Benchmark Datasets**: Preloaded with several commonly used test benchmarks, including MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, HumanEval, etc.
|
|
149
|
+
- **Evaluation Metrics**: Implements various commonly used evaluation metrics.
|
|
150
|
+
- **Model Access**: A unified model access mechanism that is compatible with the Generate and Chat interfaces of multiple model families.
|
|
151
|
+
- **Automated Evaluation**: Includes automatic evaluation of objective questions and complex task evaluation using expert models.
|
|
152
|
+
- **Evaluation Reports**: Automatically generates evaluation reports.
|
|
153
|
+
- **Arena Mode**: Used for comparisons between models and objective evaluation of models, supporting various evaluation modes, including:
|
|
154
|
+
- **Single mode**: Scoring a single model.
|
|
155
|
+
- **Pairwise-baseline mode**: Comparing against a baseline model.
|
|
156
|
+
- **Pairwise (all) mode**: Pairwise comparison among all models.
|
|
157
|
+
- **Visualization Tools**: Provides intuitive displays of evaluation results.
|
|
158
|
+
- **Model Performance Evaluation**: Offers a performance testing tool for model inference services and detailed statistics, see [Model Performance Evaluation Documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html).
|
|
159
|
+
- **OpenCompass Integration**: Supports OpenCompass as the evaluation backend, providing advanced encapsulation and task simplification, allowing for easier task submission for evaluation.
|
|
160
|
+
- **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
|
|
161
|
+
- **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
|
|
162
|
+
|
|
163
|
+
### Overall Architecture
|
|
164
|
+
<p align="center">
|
|
165
|
+
<img src="docs/en/_static/images/evalscope_framework.png" width="70%">
|
|
166
|
+
<br>Fig 1. EvalScope Framework.
|
|
167
|
+
</p>
|
|
168
|
+
|
|
169
|
+
The architecture includes the following modules:
|
|
170
|
+
1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
|
|
171
|
+
2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
|
|
172
|
+
3. **Evaluation Backend**:
|
|
173
|
+
- **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
|
|
174
|
+
- **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
|
|
175
|
+
- **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
|
|
176
|
+
- **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
|
|
177
|
+
4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
|
|
178
|
+
5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
|
|
179
|
+
6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
|
|
180
|
+
|
|
181
|
+
## 🎉 News
|
|
182
|
+
- **[2024.08.09]** Simplified installation process, supporting PyPI installation for vlmeval dependencies; Optimized multi-modal models evaluation experience with pipeline that based on OpenAI API, achieving up to 10x acceleration 🚀🚀🚀
|
|
183
|
+
- **[2024.07.31]** Breaking change: The sdk name has been changed from `llmuses` to `evalscope`, please update the sdk name in your code.
|
|
184
|
+
- **[2024.07.26]** Supports **VLMEvalKit** as a third-party evaluation framework, initiating multimodal model evaluation tasks. 🔥🔥🔥
|
|
185
|
+
- **[2024.06.29]** Supports **OpenCompass** as a third-party evaluation framework. We have provided a high-level wrapper, supporting installation via pip and simplifying the evaluation task configuration. 🔥🔥🔥
|
|
186
|
+
- **[2024.06.13]** EvalScope has been updated to version 0.3.x, which supports the ModelScope SWIFT framework for LLMs evaluation. 🚀🚀🚀
|
|
187
|
+
- **[2024.06.13]** We have supported the ToolBench as a third-party evaluation backend for Agents evaluation. 🚀🚀🚀
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
## 🛠️ Installation
|
|
192
|
+
### Method 1: Install Using pip
|
|
193
|
+
We recommend using conda to manage your environment and installing dependencies with pip:
|
|
194
|
+
|
|
195
|
+
1. Create a conda environment (optional)
|
|
196
|
+
```shell
|
|
197
|
+
# It is recommended to use Python 3.10
|
|
198
|
+
conda create -n evalscope python=3.10
|
|
199
|
+
# Activate the conda environment
|
|
200
|
+
conda activate evalscope
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
2. Install dependencies using pip
|
|
204
|
+
```shell
|
|
205
|
+
pip install evalscope # Install Native backend (default)
|
|
206
|
+
# Additional options
|
|
207
|
+
pip install evalscope[opencompass] # Install OpenCompass backend
|
|
208
|
+
pip install evalscope[vlmeval] # Install VLMEvalKit backend
|
|
209
|
+
pip install evalscope[all] # Install all backends (Native, OpenCompass, VLMEvalKit)
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
> [!WARNING]
|
|
213
|
+
> As the project has been renamed to `evalscope`, for versions `v0.4.3` or earlier, you can install using the following command:
|
|
214
|
+
> ```shell
|
|
215
|
+
> pip install llmuses<=0.4.3
|
|
216
|
+
> ```
|
|
217
|
+
> To import relevant dependencies using `llmuses`:
|
|
218
|
+
> ``` python
|
|
219
|
+
> from llmuses import ...
|
|
220
|
+
> ```
|
|
221
|
+
|
|
222
|
+
### Method 2: Install from Source
|
|
223
|
+
1. Download the source code
|
|
224
|
+
```shell
|
|
225
|
+
git clone https://github.com/modelscope/evalscope.git
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
2. Install dependencies
|
|
229
|
+
```shell
|
|
230
|
+
cd evalscope/
|
|
231
|
+
pip install -e . # Install Native backend
|
|
232
|
+
# Additional options
|
|
233
|
+
pip install -e '.[opencompass]' # Install OpenCompass backend
|
|
234
|
+
pip install -e '.[vlmeval]' # Install VLMEvalKit backend
|
|
235
|
+
pip install -e '.[all]' # Install all backends (Native, OpenCompass, VLMEvalKit)
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
## 🚀 Quick Start
|
|
240
|
+
|
|
241
|
+
### 1. Simple Evaluation
|
|
242
|
+
To evaluate a model using default settings on specified datasets, follow the process below:
|
|
243
|
+
|
|
244
|
+
#### Install using pip
|
|
245
|
+
You can execute this command from any directory:
|
|
246
|
+
```bash
|
|
247
|
+
python -m evalscope.run \
|
|
248
|
+
--model qwen/Qwen2-0.5B-Instruct \
|
|
249
|
+
--template-type qwen \
|
|
250
|
+
--datasets arc
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
#### Install from source
|
|
254
|
+
Execute this command in the `evalscope` directory:
|
|
255
|
+
```bash
|
|
256
|
+
python evalscope/run.py \
|
|
257
|
+
--model qwen/Qwen2-0.5B-Instruct \
|
|
258
|
+
--template-type qwen \
|
|
259
|
+
--datasets arc
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
If prompted with `Do you wish to run the custom code? [y/N]`, please type `y`.
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
#### Basic Parameter Descriptions
|
|
266
|
+
- `--model`: Specifies the `model_id` of the model on [ModelScope](https://modelscope.cn/), allowing automatic download. For example, see the [Qwen2-0.5B-Instruct model link](https://modelscope.cn/models/qwen/Qwen2-0.5B-Instruct/summary); you can also use a local path, such as `/path/to/model`.
|
|
267
|
+
- `--template-type`: Specifies the template type corresponding to the model. Refer to the `Default Template` field in the [template table](https://swift.readthedocs.io/en/latest/LLM/Supported-models-datasets.html) for filling in this field.
|
|
268
|
+
- `--datasets`: The dataset name, allowing multiple datasets to be specified, separated by spaces; these datasets will be automatically downloaded. Refer to the [supported datasets list](#supported-datasets-list) for available options.
|
|
269
|
+
|
|
270
|
+
### 2. Parameterized Evaluation
|
|
271
|
+
If you wish to conduct a more customized evaluation, such as modifying model parameters or dataset parameters, you can use the following commands:
|
|
272
|
+
|
|
273
|
+
**Example 1:**
|
|
274
|
+
```shell
|
|
275
|
+
python evalscope/run.py \
|
|
276
|
+
--model qwen/Qwen2-0.5B-Instruct \
|
|
277
|
+
--template-type qwen \
|
|
278
|
+
--model-args revision=v1.0.2,precision=torch.float16,device_map=auto \
|
|
279
|
+
--datasets mmlu ceval \
|
|
280
|
+
--use-cache true \
|
|
281
|
+
--limit 10
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
**Example 2:**
|
|
285
|
+
```shell
|
|
286
|
+
python evalscope/run.py \
|
|
287
|
+
--model qwen/Qwen2-0.5B-Instruct \
|
|
288
|
+
--template-type qwen \
|
|
289
|
+
--generation-config do_sample=false,temperature=0.0 \
|
|
290
|
+
--datasets ceval \
|
|
291
|
+
--dataset-args '{"ceval": {"few_shot_num": 0, "few_shot_random": false}}' \
|
|
292
|
+
--limit 10
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
#### Parameter Descriptions
|
|
296
|
+
In addition to the three [basic parameters](#basic-parameter-descriptions), the other parameters are as follows:
|
|
297
|
+
- `--model-args`: Model loading parameters, separated by commas, in `key=value` format.
|
|
298
|
+
- `--generation-config`: Generation parameters, separated by commas, in `key=value` format.
|
|
299
|
+
- `do_sample`: Whether to use sampling, default is `false`.
|
|
300
|
+
- `max_new_tokens`: Maximum generation length, default is 1024.
|
|
301
|
+
- `temperature`: Sampling temperature.
|
|
302
|
+
- `top_p`: Sampling threshold.
|
|
303
|
+
- `top_k`: Sampling threshold.
|
|
304
|
+
- `--use-cache`: Whether to use local cache, default is `false`. If set to `true`, previously evaluated model and dataset combinations will not be evaluated again, and will be read directly from the local cache.
|
|
305
|
+
- `--dataset-args`: Evaluation dataset configuration parameters, provided in JSON format, where the key is the dataset name and the value is the parameter; note that these must correspond one-to-one with the values in `--datasets`.
|
|
306
|
+
- `--few_shot_num`: Number of few-shot examples.
|
|
307
|
+
- `--few_shot_random`: Whether to randomly sample few-shot data; if not specified, defaults to `true`.
|
|
308
|
+
- `--limit`: Maximum number of evaluation samples per dataset; if not specified, all will be evaluated, which is useful for quick validation.
|
|
309
|
+
|
|
310
|
+
### 3. Use the run_task Function to Submit an Evaluation Task
|
|
311
|
+
Using the `run_task` function to submit an evaluation task requires the same parameters as the command line. You need to pass a dictionary as the parameter, which includes the following fields:
|
|
312
|
+
|
|
313
|
+
#### 1. Configuration Task Dictionary Parameters
|
|
314
|
+
```python
|
|
315
|
+
import torch
|
|
316
|
+
from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
|
|
317
|
+
|
|
318
|
+
# Example
|
|
319
|
+
your_task_cfg = {
|
|
320
|
+
'model_args': {'revision': None, 'precision': torch.float16, 'device_map': 'auto'},
|
|
321
|
+
'generation_config': {'do_sample': False, 'repetition_penalty': 1.0, 'max_new_tokens': 512},
|
|
322
|
+
'dataset_args': {},
|
|
323
|
+
'dry_run': False,
|
|
324
|
+
'model': 'qwen/Qwen2-0.5B-Instruct',
|
|
325
|
+
'template_type': 'qwen',
|
|
326
|
+
'datasets': ['arc', 'hellaswag'],
|
|
327
|
+
'work_dir': DEFAULT_ROOT_CACHE_DIR,
|
|
328
|
+
'outputs': DEFAULT_ROOT_CACHE_DIR,
|
|
329
|
+
'mem_cache': False,
|
|
330
|
+
'dataset_hub': 'ModelScope',
|
|
331
|
+
'dataset_dir': DEFAULT_ROOT_CACHE_DIR,
|
|
332
|
+
'limit': 10,
|
|
333
|
+
'debug': False
|
|
334
|
+
}
|
|
335
|
+
```
|
|
336
|
+
Here, `DEFAULT_ROOT_CACHE_DIR` is set to `'~/.cache/evalscope'`.
|
|
337
|
+
|
|
338
|
+
#### 2. Execute Task with run_task
|
|
339
|
+
```python
|
|
340
|
+
from evalscope.run import run_task
|
|
341
|
+
run_task(task_cfg=your_task_cfg)
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
### Supported Datasets List
|
|
345
|
+
> [!NOTE]
|
|
346
|
+
> The framework currently supports the following datasets. If the dataset you need is not in the list, please submit an issue, or use the [OpenCompass backend](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html) for evaluation, or use the [VLMEvalKit backend](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html) for multi-modal model evaluation.
|
|
347
|
+
|
|
348
|
+
| Dataset Name | Link | Status | Note |
|
|
349
|
+
|--------------------|----------------------------------------------------------------------------------------|--------|------|
|
|
350
|
+
| `mmlu` | [mmlu](https://modelscope.cn/datasets/modelscope/mmlu/summary) | Active | |
|
|
351
|
+
| `ceval` | [ceval](https://modelscope.cn/datasets/modelscope/ceval-exam/summary) | Active | |
|
|
352
|
+
| `gsm8k` | [gsm8k](https://modelscope.cn/datasets/modelscope/gsm8k/summary) | Active | |
|
|
353
|
+
| `arc` | [arc](https://modelscope.cn/datasets/modelscope/ai2_arc/summary) | Active | |
|
|
354
|
+
| `hellaswag` | [hellaswag](https://modelscope.cn/datasets/modelscope/hellaswag/summary) | Active | |
|
|
355
|
+
| `truthful_qa` | [truthful_qa](https://modelscope.cn/datasets/modelscope/truthful_qa/summary) | Active | |
|
|
356
|
+
| `competition_math` | [competition_math](https://modelscope.cn/datasets/modelscope/competition_math/summary) | Active | |
|
|
357
|
+
| `humaneval` | [humaneval](https://modelscope.cn/datasets/modelscope/humaneval/summary) | Active | |
|
|
358
|
+
| `bbh` | [bbh](https://modelscope.cn/datasets/modelscope/bbh/summary) | Active | |
|
|
359
|
+
| `race` | [race](https://modelscope.cn/datasets/modelscope/race/summary) | Active | |
|
|
360
|
+
| `trivia_qa` | [trivia_qa](https://modelscope.cn/datasets/modelscope/trivia_qa/summary) | To be integrated | |
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
## Evaluation Backend
|
|
364
|
+
EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:
|
|
365
|
+
- **Native**: EvalScope's own **default evaluation framework**, supporting various evaluation modes including single model evaluation, arena mode, and baseline model comparison mode.
|
|
366
|
+
- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html)
|
|
367
|
+
- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html)
|
|
368
|
+
- **ThirdParty**: The third-party task, e.g. [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html), you can contribute your own evaluation task to EvalScope as third-party backend.
|
|
369
|
+
|
|
370
|
+
## Custom Dataset Evaluation
|
|
371
|
+
EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
|
|
372
|
+
|
|
373
|
+
## Offline Evaluation
|
|
374
|
+
You can use local dataset to evaluate the model without internet connection.
|
|
375
|
+
|
|
376
|
+
Refer to: Offline Evaluation [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/offline_evaluation.html)
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
## Arena Mode
|
|
380
|
+
The Arena mode allows multiple candidate models to be evaluated through pairwise battles, and can choose to use the AI Enhanced Auto-Reviewer (AAR) automatic evaluation process or manual evaluation to obtain the evaluation report.
|
|
381
|
+
|
|
382
|
+
Refer to: Arena Mode [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html)
|
|
383
|
+
|
|
384
|
+
## Model Serving Performance Evaluation
|
|
385
|
+
A stress testing tool that focuses on large language models and can be customized to support various data set formats and different API protocol formats.
|
|
386
|
+
|
|
387
|
+
Refer to : Model Serving Performance Evaluation [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
## Leaderboard
|
|
391
|
+
The LLM Leaderboard aims to provide an objective and comprehensive evaluation standard and platform to help researchers and developers understand and compare the performance of models on various tasks on ModelScope.
|
|
392
|
+
|
|
393
|
+
Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
## TO-DO List
|
|
397
|
+
- [x] Agents evaluation
|
|
398
|
+
- [x] vLLM
|
|
399
|
+
- [ ] Distributed evaluating
|
|
400
|
+
- [x] Multi-modal evaluation
|
|
401
|
+
- [ ] Benchmarks
|
|
402
|
+
- [ ] GAIA
|
|
403
|
+
- [ ] GPQA
|
|
404
|
+
- [x] MBPP
|
|
405
|
+
- [ ] Auto-reviewer
|
|
406
|
+
- [ ] Qwen-max
|
|
407
|
+
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
evalscope/__init__.py,sha256=3eLMMrjkAIAs3vGluXNZn5-xTSbO_vfba9yNPbkVtg8,105
|
|
2
2
|
evalscope/cache.py,sha256=zpGjL9JMosqjk_dkODVwvIGiUC0WAMmMTHDNJOvBQU8,3288
|
|
3
|
-
evalscope/config.py,sha256=
|
|
3
|
+
evalscope/config.py,sha256=G_rpSn5Kd1aPlFJO6asnZu5FUggZmwcYdAxxpuq0yDs,6972
|
|
4
4
|
evalscope/constants.py,sha256=g8lGYlpA4Wk88HwtqId1-jJX_z8Lr2k02gWLsyofyj0,2670
|
|
5
5
|
evalscope/run.py,sha256=T-2zoJpBx6YxLnLJH-iFF3UxUGYTU36PMV_DQ9e8tSM,18484
|
|
6
|
-
evalscope/run_arena.py,sha256=
|
|
6
|
+
evalscope/run_arena.py,sha256=BCWCAiX0BQ9pLMIq08svEcd-IoFr75gFShpV88robIY,8963
|
|
7
7
|
evalscope/run_ms.py,sha256=UtJoGnah64SXigTawJQWTi_TEGjr7Td0rjCTaO-htL8,6028
|
|
8
|
-
evalscope/summarizer.py,sha256=
|
|
9
|
-
evalscope/version.py,sha256=
|
|
8
|
+
evalscope/summarizer.py,sha256=rIyML8HpjQxIpXg8KvQ0CzOS6xMS-JHZh6kUZzkaRsk,6640
|
|
9
|
+
evalscope/version.py,sha256=0WQd7LO3Ug6-wMC2jG2UmV0H5mWaZ-7KHtoHQB-djLc,118
|
|
10
10
|
evalscope/backend/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
|
|
11
11
|
evalscope/backend/base.py,sha256=5BLrDNNwxsGp35zorD-kphmN15tlBbkuuqwkz8jWZq0,876
|
|
12
12
|
evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
|
|
@@ -14,9 +14,10 @@ evalscope/backend/opencompass/api_meta_template.py,sha256=sBW0XbVDOKeJ7mVUDLhmcG
|
|
|
14
14
|
evalscope/backend/opencompass/backend_manager.py,sha256=Rr8eFFDUXTxI8AMcrbFW9LZuSQVZ7tsgHcZ1veNhfWM,10190
|
|
15
15
|
evalscope/backend/opencompass/tasks/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
|
|
16
16
|
evalscope/backend/opencompass/tasks/eval_api.py,sha256=12lrgDpMzZ1XBRboq5TEOovDPCMDwwGCJoRT78Ox_yo,1108
|
|
17
|
-
evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=
|
|
17
|
+
evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=EizugDMt-ontWsTOaM61XGLUkx-S9rzdLf2Ssfmw3Yc,5263
|
|
18
18
|
evalscope/backend/vlm_eval_kit/__init__.py,sha256=xTgHM95lWzh4s0W7zxLwYkgUbPAZfAb0UoGGmyyBXrs,83
|
|
19
|
-
evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=
|
|
19
|
+
evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=ZQ1uyaHxLgjrrmbXepSCluvXudHlJycibs97Js1gg_o,6125
|
|
20
|
+
evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=zC40Jw9bIqcGKuWS9oKPAlQdBARc-zY3sJlSiU-u-sI,1625
|
|
20
21
|
evalscope/benchmarks/__init__.py,sha256=6TKP35wfKf7R_h870fsEtcIlIAgomKOcukNL9M-5I1Y,162
|
|
21
22
|
evalscope/benchmarks/benchmark.py,sha256=e7rA8Y_vo6q5BhlUbZGWfZ1-SfJnU2IFRg62pnjQtDk,2157
|
|
22
23
|
evalscope/benchmarks/data_adapter.py,sha256=eVQvOQYQOQbIl8UlvOEUqRThL3FP3aUD6DSlqF1bqO0,10395
|
|
@@ -104,7 +105,7 @@ evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=xSLis-zx1hnHuj_9JI7
|
|
|
104
105
|
evalscope/models/__init__.py,sha256=zG27J2HSeKPGiAIUE7QLPHEPLyXLsfaDwYI_TDXjpCg,145
|
|
105
106
|
evalscope/models/dummy_chat_model.py,sha256=xE8wcFVSCkvizEJ-B8ojX0Ir01Q5KrN5mapjMQaQtbg,1325
|
|
106
107
|
evalscope/models/model.py,sha256=ZzzVzZHVzuzdt5F1r-rEBT44ZfW9B7R1spsrV-T8nSw,3020
|
|
107
|
-
evalscope/models/model_adapter.py,sha256=
|
|
108
|
+
evalscope/models/model_adapter.py,sha256=Cgs68ajRwTETEo1eU-OhFiFGuSx4eS1p7-JT3jOpcOk,22740
|
|
108
109
|
evalscope/models/openai_model.py,sha256=PoQS1FIiWIxp1xBJPV7Bq81LFD9FIT3vAHUvNa22DCc,3452
|
|
109
110
|
evalscope/models/template.py,sha256=Yk7-QnvjiLD0zchSZcaDSLmpW8onIeFpngSwtUOYVPk,56035
|
|
110
111
|
evalscope/models/custom/__init__.py,sha256=K4Ewo7Qrs73-jBuPq4ffxd8hMnttKhic-Zj0amH3wiU,103
|
|
@@ -158,8 +159,8 @@ evalscope/utils/logger.py,sha256=Ycd0W17Z_oiByPuPX3_umNrOCHjT9O_e_Kws7ZWUSvU,185
|
|
|
158
159
|
evalscope/utils/task_cfg_parser.py,sha256=LiNQ2X8lbZU0cODpaY_PbKyUhNoxZIC495UsLJigX64,138
|
|
159
160
|
evalscope/utils/task_utils.py,sha256=Mv_u_f4Z91zcUeko6acZCmnOAPRfk61kf_dliLzG5Yk,459
|
|
160
161
|
evalscope/utils/utils.py,sha256=zHo9hfxGBUVKE2xNMR7lDoEvfRnk4V4946DEfXQhlq4,20509
|
|
161
|
-
evalscope-0.5.
|
|
162
|
-
evalscope-0.5.
|
|
163
|
-
evalscope-0.5.
|
|
164
|
-
evalscope-0.5.
|
|
165
|
-
evalscope-0.5.
|
|
162
|
+
evalscope-0.5.3.dist-info/METADATA,sha256=19GatH8y-jNjQbVX-IGuRb1g2VTjPBOs2dh9RVqrCCQ,21835
|
|
163
|
+
evalscope-0.5.3.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
164
|
+
evalscope-0.5.3.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
|
|
165
|
+
evalscope-0.5.3.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
|
|
166
|
+
evalscope-0.5.3.dist-info/RECORD,,
|
|
@@ -1,578 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: evalscope
|
|
3
|
-
Version: 0.5.2
|
|
4
|
-
Summary: EvalScope: Lightweight LLMs Evaluation Framework
|
|
5
|
-
Home-page: https://github.com/modelscope/evalscope
|
|
6
|
-
Author: ModelScope team
|
|
7
|
-
Author-email: contact@modelscope.cn
|
|
8
|
-
Keywords: python,llm,evaluation
|
|
9
|
-
Classifier: Development Status :: 4 - Beta
|
|
10
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
-
Classifier: Operating System :: OS Independent
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
-
Requires-Python: >=3.8
|
|
17
|
-
Description-Content-Type: text/markdown
|
|
18
|
-
Requires-Dist: torch
|
|
19
|
-
Requires-Dist: absl-py
|
|
20
|
-
Requires-Dist: accelerate
|
|
21
|
-
Requires-Dist: cachetools
|
|
22
|
-
Requires-Dist: editdistance
|
|
23
|
-
Requires-Dist: jsonlines
|
|
24
|
-
Requires-Dist: matplotlib
|
|
25
|
-
Requires-Dist: modelscope[framework]
|
|
26
|
-
Requires-Dist: nltk
|
|
27
|
-
Requires-Dist: openai
|
|
28
|
-
Requires-Dist: pandas
|
|
29
|
-
Requires-Dist: plotly
|
|
30
|
-
Requires-Dist: pyarrow
|
|
31
|
-
Requires-Dist: pympler
|
|
32
|
-
Requires-Dist: pyyaml
|
|
33
|
-
Requires-Dist: regex
|
|
34
|
-
Requires-Dist: requests
|
|
35
|
-
Requires-Dist: requests-toolbelt
|
|
36
|
-
Requires-Dist: rouge-score
|
|
37
|
-
Requires-Dist: sacrebleu
|
|
38
|
-
Requires-Dist: scikit-learn
|
|
39
|
-
Requires-Dist: seaborn
|
|
40
|
-
Requires-Dist: sentencepiece
|
|
41
|
-
Requires-Dist: simple-ddl-parser
|
|
42
|
-
Requires-Dist: tabulate
|
|
43
|
-
Requires-Dist: tiktoken
|
|
44
|
-
Requires-Dist: tqdm
|
|
45
|
-
Requires-Dist: transformers (<4.43,>=4.33)
|
|
46
|
-
Requires-Dist: transformers-stream-generator
|
|
47
|
-
Requires-Dist: jieba
|
|
48
|
-
Requires-Dist: rouge-chinese
|
|
49
|
-
Provides-Extra: all
|
|
50
|
-
Requires-Dist: torch ; extra == 'all'
|
|
51
|
-
Requires-Dist: absl-py ; extra == 'all'
|
|
52
|
-
Requires-Dist: accelerate ; extra == 'all'
|
|
53
|
-
Requires-Dist: cachetools ; extra == 'all'
|
|
54
|
-
Requires-Dist: editdistance ; extra == 'all'
|
|
55
|
-
Requires-Dist: jsonlines ; extra == 'all'
|
|
56
|
-
Requires-Dist: matplotlib ; extra == 'all'
|
|
57
|
-
Requires-Dist: modelscope[framework] ; extra == 'all'
|
|
58
|
-
Requires-Dist: nltk ; extra == 'all'
|
|
59
|
-
Requires-Dist: openai ; extra == 'all'
|
|
60
|
-
Requires-Dist: pandas ; extra == 'all'
|
|
61
|
-
Requires-Dist: plotly ; extra == 'all'
|
|
62
|
-
Requires-Dist: pyarrow ; extra == 'all'
|
|
63
|
-
Requires-Dist: pympler ; extra == 'all'
|
|
64
|
-
Requires-Dist: pyyaml ; extra == 'all'
|
|
65
|
-
Requires-Dist: regex ; extra == 'all'
|
|
66
|
-
Requires-Dist: requests ; extra == 'all'
|
|
67
|
-
Requires-Dist: requests-toolbelt ; extra == 'all'
|
|
68
|
-
Requires-Dist: rouge-score ; extra == 'all'
|
|
69
|
-
Requires-Dist: sacrebleu ; extra == 'all'
|
|
70
|
-
Requires-Dist: scikit-learn ; extra == 'all'
|
|
71
|
-
Requires-Dist: seaborn ; extra == 'all'
|
|
72
|
-
Requires-Dist: sentencepiece ; extra == 'all'
|
|
73
|
-
Requires-Dist: simple-ddl-parser ; extra == 'all'
|
|
74
|
-
Requires-Dist: tabulate ; extra == 'all'
|
|
75
|
-
Requires-Dist: tiktoken ; extra == 'all'
|
|
76
|
-
Requires-Dist: tqdm ; extra == 'all'
|
|
77
|
-
Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'all'
|
|
78
|
-
Requires-Dist: transformers-stream-generator ; extra == 'all'
|
|
79
|
-
Requires-Dist: jieba ; extra == 'all'
|
|
80
|
-
Requires-Dist: rouge-chinese ; extra == 'all'
|
|
81
|
-
Requires-Dist: ms-opencompass (>=0.0.5) ; extra == 'all'
|
|
82
|
-
Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'all'
|
|
83
|
-
Provides-Extra: inner
|
|
84
|
-
Requires-Dist: absl-py ; extra == 'inner'
|
|
85
|
-
Requires-Dist: accelerate ; extra == 'inner'
|
|
86
|
-
Requires-Dist: alibaba-itag-sdk ; extra == 'inner'
|
|
87
|
-
Requires-Dist: dashscope ; extra == 'inner'
|
|
88
|
-
Requires-Dist: editdistance ; extra == 'inner'
|
|
89
|
-
Requires-Dist: jsonlines ; extra == 'inner'
|
|
90
|
-
Requires-Dist: nltk ; extra == 'inner'
|
|
91
|
-
Requires-Dist: openai ; extra == 'inner'
|
|
92
|
-
Requires-Dist: pandas (==1.5.3) ; extra == 'inner'
|
|
93
|
-
Requires-Dist: plotly ; extra == 'inner'
|
|
94
|
-
Requires-Dist: pyarrow ; extra == 'inner'
|
|
95
|
-
Requires-Dist: pyodps ; extra == 'inner'
|
|
96
|
-
Requires-Dist: pyyaml ; extra == 'inner'
|
|
97
|
-
Requires-Dist: regex ; extra == 'inner'
|
|
98
|
-
Requires-Dist: requests (==2.28.1) ; extra == 'inner'
|
|
99
|
-
Requires-Dist: requests-toolbelt (==0.10.1) ; extra == 'inner'
|
|
100
|
-
Requires-Dist: rouge-score ; extra == 'inner'
|
|
101
|
-
Requires-Dist: sacrebleu ; extra == 'inner'
|
|
102
|
-
Requires-Dist: scikit-learn ; extra == 'inner'
|
|
103
|
-
Requires-Dist: seaborn ; extra == 'inner'
|
|
104
|
-
Requires-Dist: simple-ddl-parser ; extra == 'inner'
|
|
105
|
-
Requires-Dist: streamlit ; extra == 'inner'
|
|
106
|
-
Requires-Dist: tqdm ; extra == 'inner'
|
|
107
|
-
Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'inner'
|
|
108
|
-
Requires-Dist: transformers-stream-generator ; extra == 'inner'
|
|
109
|
-
Provides-Extra: opencompass
|
|
110
|
-
Requires-Dist: ms-opencompass (>=0.0.5) ; extra == 'opencompass'
|
|
111
|
-
Provides-Extra: vlmeval
|
|
112
|
-
Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'vlmeval'
|
|
113
|
-
|
|
114
|
-
English | [简体中文](README_zh.md)
|
|
115
|
-
|
|
116
|
-
<p align="center">
|
|
117
|
-
<a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/evalscope">
|
|
118
|
-
</a>
|
|
119
|
-
<a href="https://github.com/modelscope/evalscope/pulls"><img src="https://img.shields.io/badge/PR-welcome-55EB99.svg"></a>
|
|
120
|
-
<p>
|
|
121
|
-
|
|
122
|
-
## 📖 Table of Content
|
|
123
|
-
- [Introduction](#introduction)
|
|
124
|
-
- [News](#News)
|
|
125
|
-
- [Installation](#installation)
|
|
126
|
-
- [Quick Start](#quick-start)
|
|
127
|
-
- [Dataset List](#datasets-list)
|
|
128
|
-
- [Leaderboard](#leaderboard)
|
|
129
|
-
- [Experiments and Results](#Experiments-and-Results)
|
|
130
|
-
- [Model Serving Performance Evaluation](#Model-Serving-Performance-Evaluation)
|
|
131
|
-
|
|
132
|
-
## 📝 Introduction
|
|
133
|
-
|
|
134
|
-
Large Language Model (LLMs) evaluation has become a critical process for assessing and improving LLMs. To better support the evaluation of large models, we propose the EvalScope framework, which includes the following components and features:
|
|
135
|
-
|
|
136
|
-
- Pre-configured common benchmark datasets, including: MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, HumanEval, etc.
|
|
137
|
-
- Implementation of common evaluation metrics
|
|
138
|
-
- Unified model integration, compatible with the generate and chat interfaces of multiple model series
|
|
139
|
-
- Automatic evaluation (evaluator):
|
|
140
|
-
- Automatic evaluation for objective questions
|
|
141
|
-
- Implementation of complex task evaluation using expert models
|
|
142
|
-
- Reports of evaluation generating
|
|
143
|
-
- Arena mode
|
|
144
|
-
- Visualization tools
|
|
145
|
-
- Model Inference Performance Evaluation [Tutorial](evalscope/perf/README.md)
|
|
146
|
-
- Support for OpenCompass as an Evaluation Backend, featuring advanced encapsulation and task simplification to easily submit tasks to OpenCompass for evaluation.
|
|
147
|
-
- Supports VLMEvalKit as the evaluation backend. It initiates VLMEvalKit's multimodal evaluation tasks through EvalScope, supporting various multimodal models and datasets.
|
|
148
|
-
- Full pipeline support: Seamlessly integrate with SWIFT to easily train and deploy model services, initiate evaluation tasks, view evaluation reports, and achieve an end-to-end large model development process.
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
**Features**
|
|
152
|
-
- Lightweight, minimizing unnecessary abstractions and configurations
|
|
153
|
-
- Easy to customize
|
|
154
|
-
- New datasets can be integrated by simply implementing a single class
|
|
155
|
-
- Models can be hosted on [ModelScope](https://modelscope.cn), and evaluations can be initiated with just a model id
|
|
156
|
-
- Supports deployment of locally hosted models
|
|
157
|
-
- Visualization of evaluation reports
|
|
158
|
-
- Rich evaluation metrics
|
|
159
|
-
- Model-based automatic evaluation process, supporting multiple evaluation modes
|
|
160
|
-
- Single mode: Expert models score individual models
|
|
161
|
-
- Pairwise-baseline mode: Comparison with baseline models
|
|
162
|
-
- Pairwise (all) mode: Pairwise comparison of all models
|
|
163
|
-
|
|
164
|
-
## 🎉 News
|
|
165
|
-
- **[2024.07.31]** Breaking change: The sdk name has been changed from `llmuses` to `evalscope`, please update the sdk name in your code.
|
|
166
|
-
- **[2024.07.26]** Supports **VLMEvalKit** as a third-party evaluation framework, initiating multimodal model evaluation tasks. [User Guide](#vlmevalkit-evaluation-backend) 🔥🔥🔥
|
|
167
|
-
- **[2024.06.29]** Supports **OpenCompass** as a third-party evaluation framework. We have provided a high-level wrapper, supporting installation via pip and simplifying the evaluation task configuration. [User Guide](#opencompass-evaluation-backend) 🔥🔥🔥
|
|
168
|
-
- **[2024.06.13]** EvalScope has been updated to version 0.3.x, which supports the ModelScope SWIFT framework for LLMs evaluation. 🚀🚀🚀
|
|
169
|
-
- **[2024.06.13]** We have supported the ToolBench as a third-party evaluation backend for Agents evaluation. 🚀🚀🚀
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
## 🛠️ Installation
|
|
174
|
-
### Install with pip
|
|
175
|
-
1. create conda environment [Optional]
|
|
176
|
-
```shell
|
|
177
|
-
conda create -n evalscope python=3.10
|
|
178
|
-
conda activate evalscope
|
|
179
|
-
```
|
|
180
|
-
|
|
181
|
-
2. Install EvalScope
|
|
182
|
-
```shell
|
|
183
|
-
pip install evalscope # Installation with Native backend (by default)
|
|
184
|
-
|
|
185
|
-
pip install evalscope[opencompass] # Installation with OpenCompass backend
|
|
186
|
-
pip install evalscope[vlmeval] # Installation with VLMEvalKit backend
|
|
187
|
-
pip install evalscope[all] # Installation with all backends (Native, OpenCompass, VLMEvalKit)
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
DEPRECATION WARNING: For 0.4.3 or older versions, please use the following command to install:
|
|
191
|
-
```shell
|
|
192
|
-
pip install llmuses<=0.4.3
|
|
193
|
-
|
|
194
|
-
# Usage:
|
|
195
|
-
from llmuses.run import run_task
|
|
196
|
-
...
|
|
197
|
-
|
|
198
|
-
```
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
### Install from source code
|
|
202
|
-
1. Download source code
|
|
203
|
-
```shell
|
|
204
|
-
git clone https://github.com/modelscope/evalscope.git
|
|
205
|
-
```
|
|
206
|
-
|
|
207
|
-
2. Install dependencies
|
|
208
|
-
```shell
|
|
209
|
-
cd evalscope/
|
|
210
|
-
pip install -e .
|
|
211
|
-
```
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
## 🚀 Quick Start
|
|
215
|
-
|
|
216
|
-
### Simple Evaluation
|
|
217
|
-
command line with pip installation:
|
|
218
|
-
```shell
|
|
219
|
-
python -m evalscope.run --model ZhipuAI/chatglm3-6b --template-type chatglm3 --datasets arc --limit 100
|
|
220
|
-
```
|
|
221
|
-
command line with source code:
|
|
222
|
-
```shell
|
|
223
|
-
python evalscope/run.py --model ZhipuAI/chatglm3-6b --template-type chatglm3 --datasets mmlu ceval --limit 10
|
|
224
|
-
```
|
|
225
|
-
Parameters:
|
|
226
|
-
- --model: ModelScope model id, model link: [ZhipuAI/chatglm3-6b](https://modelscope.cn/models/ZhipuAI/chatglm3-6b/summary)
|
|
227
|
-
|
|
228
|
-
### Evaluation with Model Arguments
|
|
229
|
-
```shell
|
|
230
|
-
python evalscope/run.py --model ZhipuAI/chatglm3-6b --template-type chatglm3 --model-args revision=v1.0.2,precision=torch.float16,device_map=auto --datasets mmlu ceval --use-cache true --limit 10
|
|
231
|
-
```
|
|
232
|
-
```shell
|
|
233
|
-
python evalscope/run.py --model qwen/Qwen-1_8B --generation-config do_sample=false,temperature=0.0 --datasets ceval --dataset-args '{"ceval": {"few_shot_num": 0, "few_shot_random": false}}' --limit 10
|
|
234
|
-
```
|
|
235
|
-
Parameters:
|
|
236
|
-
- --model-args: Parameters of model: revision, precision, device_map, in format of key=value,key=value
|
|
237
|
-
- --datasets: datasets list, separated by space
|
|
238
|
-
- --use-cache: `true` or `false`, whether to use cache, default is `false`
|
|
239
|
-
- --dataset-args: evaluation settings，json format，key is the dataset name，value should be args for the dataset
|
|
240
|
-
- --few_shot_num: few-shot data number
|
|
241
|
-
- --few_shot_random: whether to use random few-shot data, default is `true`
|
|
242
|
-
- --local_path: local dataset path
|
|
243
|
-
- --limit: maximum number of samples to evaluate for each sub-dataset
|
|
244
|
-
- --template-type: model template type, see [Template Type List](https://github.com/modelscope/swift/blob/main/docs/source_en/LLM/Supported-models-datasets.md)
|
|
245
|
-
|
|
246
|
-
Note: you can use following command to check the template type list of the model:
|
|
247
|
-
```shell
|
|
248
|
-
from evalscope.models.template import TemplateType
|
|
249
|
-
print(TemplateType.get_template_name_list())
|
|
250
|
-
```
|
|
251
|
-
|
|
252
|
-
### Evaluation Backend
|
|
253
|
-
EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:
|
|
254
|
-
- **Native**: EvalScope's own **default evaluation framework**, supporting various evaluation modes including single model evaluation, arena mode, and baseline model comparison mode.
|
|
255
|
-
- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework [ModelScope Swift](https://github.com/modelscope/swift).
|
|
256
|
-
- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework [ModelScope Swift](https://github.com/modelscope/swift).
|
|
257
|
-
- **ThirdParty**: The third-party task, e.g. [ToolBench](evalscope/thirdparty/toolbench/README.md), you can contribute your own evaluation task to EvalScope as third-party backend.
|
|
258
|
-
|
|
259
|
-
#### OpenCompass Eval-Backend
|
|
260
|
-
|
|
261
|
-
To facilitate the use of the OpenCompass evaluation backend, we have customized the OpenCompass source code and named it `ms-opencompass`. This version includes optimizations for evaluation task configuration and execution based on the original version, and it supports installation via PyPI. This allows users to initiate lightweight OpenCompass evaluation tasks through EvalScope. Additionally, we have initially opened up API-based evaluation tasks in the OpenAI API format. You can deploy model services using [ModelScope Swift](https://github.com/modelscope/swift), where [swift deploy](https://swift.readthedocs.io/en/latest/LLM/VLLM-inference-acceleration-and-deployment.html) supports using vLLM to launch model inference services.
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
##### Installation
|
|
265
|
-
```shell
|
|
266
|
-
# Install with extra option
|
|
267
|
-
pip install evalscope[opencompass]
|
|
268
|
-
```
|
|
269
|
-
|
|
270
|
-
##### Data Preparation
|
|
271
|
-
Available datasets from OpenCompass backend:
|
|
272
|
-
```text
|
|
273
|
-
'obqa', 'AX_b', 'siqa', 'nq', 'mbpp', 'winogrande', 'mmlu', 'BoolQ', 'cluewsc', 'ocnli', 'lambada', 'CMRC', 'ceval', 'csl', 'cmnli', 'bbh', 'ReCoRD', 'math', 'humaneval', 'eprstmt', 'WSC', 'storycloze', 'MultiRC', 'RTE', 'chid', 'gsm8k', 'AX_g', 'bustm', 'afqmc', 'piqa', 'lcsts', 'strategyqa', 'Xsum', 'agieval', 'ocnli_fc', 'C3', 'tnews', 'race', 'triviaqa', 'CB', 'WiC', 'hellaswag', 'summedits', 'GaokaoBench', 'ARC_e', 'COPA', 'ARC_c', 'DRCD'
|
|
274
|
-
```
|
|
275
|
-
Refer to [OpenCompass datasets](https://hub.opencompass.org.cn/home)
|
|
276
|
-
|
|
277
|
-
You can use the following code to list all available datasets:
|
|
278
|
-
```python
|
|
279
|
-
from evalscope.backend.opencompass import OpenCompassBackendManager
|
|
280
|
-
print(f'** All datasets from OpenCompass backend: {OpenCompassBackendManager.list_datasets()}')
|
|
281
|
-
```
|
|
282
|
-
|
|
283
|
-
Dataset download:
|
|
284
|
-
- Option1: Download from ModelScope
|
|
285
|
-
```shell
|
|
286
|
-
git clone https://www.modelscope.cn/datasets/swift/evalscope_resource.git
|
|
287
|
-
```
|
|
288
|
-
|
|
289
|
-
- Option2: Download from OpenCompass GitHub
|
|
290
|
-
```shell
|
|
291
|
-
wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-complete-20240207.zip
|
|
292
|
-
```
|
|
293
|
-
|
|
294
|
-
Unzip the file and set the path to the `data` directory in current work directory.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
##### Model Serving
|
|
298
|
-
We use ModelScope swift to deploy model services, see: [ModelScope Swift](hhttps://swift.readthedocs.io/en/latest/LLM/VLLM-inference-acceleration-and-deployment.html)
|
|
299
|
-
```shell
|
|
300
|
-
# Install ms-swift
|
|
301
|
-
pip install ms-swift
|
|
302
|
-
|
|
303
|
-
# Deploy model
|
|
304
|
-
CUDA_VISIBLE_DEVICES=0 swift deploy --model_type llama3-8b-instruct --port 8000
|
|
305
|
-
```
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
##### Model Evaluation
|
|
309
|
-
|
|
310
|
-
Refer to example: [example_eval_swift_openai_api](examples/example_eval_swift_openai_api.py) to configure and execute the evaluation task:
|
|
311
|
-
```shell
|
|
312
|
-
python examples/example_eval_swift_openai_api.py
|
|
313
|
-
```
|
|
314
|
-
|
|
315
|
-
#### VLMEvalKit Evaluation Backend
|
|
316
|
-
|
|
317
|
-
To facilitate the use of the VLMEvalKit evaluation backend, we have customized the VLMEvalKit source code and named it `ms-vlmeval`. This version encapsulates the configuration and execution of evaluation tasks based on the original version and supports installation via PyPI, allowing users to initiate lightweight VLMEvalKit evaluation tasks through EvalScope. Additionally, we support API-based evaluation tasks in the OpenAI API format. You can deploy multimodal model services using ModelScope [swift](https://github.com/modelscope/swift).
|
|
318
|
-
|
|
319
|
-
##### Installation
|
|
320
|
-
```shell
|
|
321
|
-
# Install with additional options
|
|
322
|
-
pip install evalscope[vlmeval]
|
|
323
|
-
```
|
|
324
|
-
|
|
325
|
-
##### Data Preparation
|
|
326
|
-
Currently supported datasets include:
|
|
327
|
-
```text
|
|
328
|
-
'COCO_VAL', 'MME', 'HallusionBench', 'POPE', 'MMBench_DEV_EN', 'MMBench_TEST_EN', 'MMBench_DEV_CN', 'MMBench_TEST_CN', 'MMBench', 'MMBench_CN', 'MMBench_DEV_EN_V11', 'MMBench_TEST_EN_V11', 'MMBench_DEV_CN_V11', 'MMBench_TEST_CN_V11', 'MMBench_V11', 'MMBench_CN_V11', 'SEEDBench_IMG', 'SEEDBench2', 'SEEDBench2_Plus', 'ScienceQA_VAL', 'ScienceQA_TEST', 'MMT-Bench_ALL_MI', 'MMT-Bench_ALL', 'MMT-Bench_VAL_MI', 'MMT-Bench_VAL', 'AesBench_VAL', 'AesBench_TEST', 'CCBench', 'AI2D_TEST', 'MMStar', 'RealWorldQA', 'MLLMGuard_DS', 'BLINK', 'OCRVQA_TEST', 'OCRVQA_TESTCORE', 'TextVQA_VAL', 'DocVQA_VAL', 'DocVQA_TEST', 'InfoVQA_ VAL', 'InfoVQA_TEST', 'ChartQA_VAL', 'ChartQA_TEST', 'MathVision', 'MathVision_MINI', 'MMMU_DEV_VAL', 'MMMU_TEST', 'OCRBench', 'MathVista_MINI', 'LLaVABench', 'MMVet', 'MTVQA_TEST', 'MMLongBench_DOC', 'VCR_EN_EASY_500', 'VCR_EN_EASY_100', 'VCR_EN_EASY_ALL', 'VCR_EN_HARD_500', 'VCR_EN_HARD_100', 'VCR_EN_HARD_ALL', 'VCR_ZH_EASY_500', 'VCR_ZH_EASY_100', 'VCR_Z H_EASY_ALL', 'VCR_ZH_HARD_500', 'VCR_ZH_HARD_100', 'VCR_ZH_HARD_ALL', 'MMBench-Video', 'Video-MME', 'MMBench_DEV_EN', 'MMBench_TEST_EN', 'MMBench_DEV_CN', 'MMBench_TEST_CN', 'MMBench', 'MMBench_CN', 'MMBench_DEV_EN_V11', 'MMBench_TEST_EN_V11', 'MMBench_DEV_CN_V11', 'MMBench_TEST_CN_V11', 'MM Bench_V11', 'MMBench_CN_V11', 'SEEDBench_IMG', 'SEEDBench2', 'SEEDBench2_Plus', 'ScienceQA_VAL', 'ScienceQA_TEST', 'MMT-Bench_ALL_MI', 'MMT-Bench_ALL', 'MMT-Bench_VAL_MI', 'MMT-Bench_VAL', 'AesBench_VAL', 'AesBench_TEST', 'CCBench', 'AI2D_TEST', 'MMStar', 'RealWorldQA', 'MLLMGuard_DS', 'BLINK'
|
|
329
|
-
```
|
|
330
|
-
For detailed information about the datasets, please refer to [VLMEvalKit Supported Multimodal Evaluation Sets](https://github.com/open-compass/VLMEvalKit/tree/main#-datasets-models-and-evaluation-results).
|
|
331
|
-
|
|
332
|
-
You can use the following to view the list of dataset names:
|
|
333
|
-
```python
|
|
334
|
-
from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
|
|
335
|
-
print(f'** All models from VLMEvalKit backend: {VLMEvalKitBackendManager.list_supported_models().keys()}')
|
|
336
|
-
|
|
337
|
-
```
|
|
338
|
-
If the dataset file does not exist locally when loading the dataset, it will be automatically downloaded to the `~/LMUData/` directory.
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
##### Model Evaluation
|
|
342
|
-
There are two ways to evaluate the model:
|
|
343
|
-
|
|
344
|
-
###### 1. ModelScope Swift Deployment for Model Evaluation
|
|
345
|
-
**Model Deployment**
|
|
346
|
-
Deploy the model service using ModelScope Swift. For detailed instructions, refer to: [ModelScope Swift MLLM Deployment Guide](https://swift.readthedocs.io/en/latest/Multi-Modal/mutlimodal-deployment.html)
|
|
347
|
-
```shell
|
|
348
|
-
# Install ms-swift
|
|
349
|
-
pip install ms-swift
|
|
350
|
-
# Deploy the qwen-vl-chat multi-modal model service
|
|
351
|
-
CUDA_VISIBLE_DEVICES=0 swift deploy --model_type qwen-vl-chat --model_id_or_path models/Qwen-VL-Chat
|
|
352
|
-
```
|
|
353
|
-
**Model Evaluation**
|
|
354
|
-
Refer to the example file: [example_eval_vlm_swift](examples/example_eval_vlm_swift.py) to configure the evaluation task.
|
|
355
|
-
Execute the evaluation task:
|
|
356
|
-
```shell
|
|
357
|
-
python examples/example_eval_vlm_swift.py
|
|
358
|
-
```
|
|
359
|
-
|
|
360
|
-
###### 2. Local Model Inference Evaluation
|
|
361
|
-
**Model Inference Evaluation**
|
|
362
|
-
Skip the model service deployment and perform inference directly on the local machine. Refer to the example file: [example_eval_vlm_local](examples/example_eval_vlm_local.py) to configure the evaluation task.
|
|
363
|
-
Execute the evaluation task:
|
|
364
|
-
```shell
|
|
365
|
-
python examples/example_eval_vlm_local.py
|
|
366
|
-
```
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
##### (Optional) Deploy Judge Model
|
|
370
|
-
Deploy the local language model as a judge/extractor using ModelScope swift. For details, refer to: [ModelScope Swift LLM Deployment Guide](https://swift.readthedocs.io/en/latest/LLM/VLLM-inference-acceleration-and-deployment.html). If no judge model is deployed, exact matching will be used.
|
|
371
|
-
|
|
372
|
-
```shell
|
|
373
|
-
# Deploy qwen2-7b as a judge
|
|
374
|
-
CUDA_VISIBLE_DEVICES=1 swift deploy --model_type qwen2-7b-instruct --model_id_or_path models/Qwen2-7B-Instruct --port 8866
|
|
375
|
-
```
|
|
376
|
-
|
|
377
|
-
You **must configure the following environment variables for the judge model to be correctly invoked**:
|
|
378
|
-
```
|
|
379
|
-
OPENAI_API_KEY=EMPTY
|
|
380
|
-
OPENAI_API_BASE=http://127.0.0.1:8866/v1/chat/completions # api_base for the judge model
|
|
381
|
-
LOCAL_LLM=qwen2-7b-instruct # model_id for the judge model
|
|
382
|
-
```
|
|
383
|
-
|
|
384
|
-
##### Model Evaluation
|
|
385
|
-
Refer to the example file: [example_eval_vlm_swift](examples/example_eval_vlm_swift.py) to configure the evaluation task.
|
|
386
|
-
|
|
387
|
-
Execute the evaluation task:
|
|
388
|
-
|
|
389
|
-
```shell
|
|
390
|
-
python examples/example_eval_vlm_swift.py
|
|
391
|
-
```
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
### Local Dataset
|
|
395
|
-
You can use local dataset to evaluate the model without internet connection.
|
|
396
|
-
#### 1. Download and unzip the dataset
|
|
397
|
-
```shell
|
|
398
|
-
# set path to /path/to/workdir
|
|
399
|
-
wget https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/benchmark/data.zip
|
|
400
|
-
unzip data.zip
|
|
401
|
-
```
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
#### 2. Use local dataset to evaluate the model
|
|
405
|
-
```shell
|
|
406
|
-
python evalscope/run.py --model ZhipuAI/chatglm3-6b --template-type chatglm3 --datasets arc --dataset-hub Local --dataset-args '{"arc": {"local_path": "/path/to/workdir/data/arc"}}' --limit 10
|
|
407
|
-
|
|
408
|
-
# Parameters:
|
|
409
|
-
# --dataset-hub: dataset sources: `ModelScope`, `Local`, `HuggingFace` (TO-DO) default to `ModelScope`
|
|
410
|
-
# --dataset-args: json format, key is the dataset name, value should be args for the dataset
|
|
411
|
-
```
|
|
412
|
-
|
|
413
|
-
#### 3. (Optional) Use local mode to submit evaluation task
|
|
414
|
-
|
|
415
|
-
```shell
|
|
416
|
-
# 1. Prepare the model local folder, the folder structure refers to chatglm3-6b, link: https://modelscope.cn/models/ZhipuAI/chatglm3-6b/files
|
|
417
|
-
# For example, download the model folder to the local path /path/to/ZhipuAI/chatglm3-6b
|
|
418
|
-
|
|
419
|
-
# 2. Execute the offline evaluation task
|
|
420
|
-
python evalscope/run.py --model /path/to/ZhipuAI/chatglm3-6b --template-type chatglm3 --datasets arc --dataset-hub Local --dataset-args '{"arc": {"local_path": "/path/to/workdir/data/arc"}}' --limit 10
|
|
421
|
-
```
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
### Use run_task function
|
|
425
|
-
|
|
426
|
-
#### 1. Configuration
|
|
427
|
-
```python
|
|
428
|
-
import torch
|
|
429
|
-
from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
|
|
430
|
-
|
|
431
|
-
# Example configuration
|
|
432
|
-
your_task_cfg = {
|
|
433
|
-
'model_args': {'revision': None, 'precision': torch.float16, 'device_map': 'auto'},
|
|
434
|
-
'generation_config': {'do_sample': False, 'repetition_penalty': 1.0, 'max_new_tokens': 512},
|
|
435
|
-
'dataset_args': {},
|
|
436
|
-
'dry_run': False,
|
|
437
|
-
'model': 'ZhipuAI/chatglm3-6b',
|
|
438
|
-
'template_type': 'chatglm3',
|
|
439
|
-
'datasets': ['arc', 'hellaswag'],
|
|
440
|
-
'work_dir': DEFAULT_ROOT_CACHE_DIR,
|
|
441
|
-
'outputs': DEFAULT_ROOT_CACHE_DIR,
|
|
442
|
-
'mem_cache': False,
|
|
443
|
-
'dataset_hub': 'ModelScope',
|
|
444
|
-
'dataset_dir': DEFAULT_ROOT_CACHE_DIR,
|
|
445
|
-
'stage': 'all',
|
|
446
|
-
'limit': 10,
|
|
447
|
-
'debug': False
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
```
|
|
451
|
-
|
|
452
|
-
#### 2. Execute the task
|
|
453
|
-
```python
|
|
454
|
-
from evalscope.run import run_task
|
|
455
|
-
|
|
456
|
-
run_task(task_cfg=your_task_cfg)
|
|
457
|
-
```
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
### Arena Mode
|
|
461
|
-
The Arena mode allows multiple candidate models to be evaluated through pairwise battles, and can choose to use the AI Enhanced Auto-Reviewer (AAR) automatic evaluation process or manual evaluation to obtain the evaluation report. The process is as follows:
|
|
462
|
-
#### 1. Env preparation
|
|
463
|
-
```text
|
|
464
|
-
a. Data preparation, the question data format refers to: evalscope/registry/data/question.jsonl
|
|
465
|
-
b. If you need to use the automatic evaluation process (AAR), you need to configure the relevant environment variables. Taking the GPT-4 based auto-reviewer process as an example, you need to configure the following environment variables:
|
|
466
|
-
> export OPENAI_API_KEY=YOUR_OPENAI_API_KEY
|
|
467
|
-
```
|
|
468
|
-
|
|
469
|
-
#### 2. Configuration files
|
|
470
|
-
```text
|
|
471
|
-
Refer to : evalscope/registry/config/cfg_arena.yaml
|
|
472
|
-
Parameters:
|
|
473
|
-
questions_file: question data path
|
|
474
|
-
answers_gen: candidate model prediction result generation, supports multiple models, can control whether to enable the model through the enable parameter
|
|
475
|
-
reviews_gen: evaluation result generation, currently defaults to using GPT-4 as the Auto-reviewer, can control whether to enable this step through the enable parameter
|
|
476
|
-
elo_rating: ELO rating algorithm, can control whether to enable this step through the enable parameter, note that this step depends on the review_file must exist
|
|
477
|
-
```
|
|
478
|
-
|
|
479
|
-
#### 3. Execute the script
|
|
480
|
-
```shell
|
|
481
|
-
#Usage:
|
|
482
|
-
cd evalscope
|
|
483
|
-
|
|
484
|
-
# dry-run mode
|
|
485
|
-
python evalscope/run_arena.py -c registry/config/cfg_arena.yaml --dry-run
|
|
486
|
-
|
|
487
|
-
# Execute the script
|
|
488
|
-
python evalscope/run_arena.py --c registry/config/cfg_arena.yaml
|
|
489
|
-
```
|
|
490
|
-
|
|
491
|
-
#### 4. Visualization
|
|
492
|
-
|
|
493
|
-
```shell
|
|
494
|
-
# Usage:
|
|
495
|
-
streamlit run viz.py -- --review-file evalscope/registry/data/qa_browser/battle.jsonl --category-file evalscope/registry/data/qa_browser/category_mapping.yaml
|
|
496
|
-
```
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
### Single Model Evaluation Mode
|
|
500
|
-
|
|
501
|
-
In this mode, we only score the output of a single model, without pairwise comparison.
|
|
502
|
-
#### 1. Configuration file
|
|
503
|
-
```text
|
|
504
|
-
Refer to: evalscope/registry/config/cfg_single.yaml
|
|
505
|
-
Parameters:
|
|
506
|
-
questions_file: question data path
|
|
507
|
-
answers_gen: candidate model prediction result generation, supports multiple models, can control whether to enable the model through the enable parameter
|
|
508
|
-
reviews_gen: evaluation result generation, currently defaults to using GPT-4 as the Auto-reviewer, can control whether to enable this step through the enable parameter
|
|
509
|
-
rating_gen: rating algorithm, can control whether to enable this step through the enable parameter, note that this step depends on the review_file must exist
|
|
510
|
-
```
|
|
511
|
-
#### 2. Execute the script
|
|
512
|
-
```shell
|
|
513
|
-
#Example:
|
|
514
|
-
python evalscope/run_arena.py --c registry/config/cfg_single.yaml
|
|
515
|
-
```
|
|
516
|
-
|
|
517
|
-
### Baseline Model Comparison Mode
|
|
518
|
-
|
|
519
|
-
In this mode, we select the baseline model, and compare other models with the baseline model for scoring. This mode can easily add new models to the Leaderboard (just need to run the scoring with the new model and the baseline model).
|
|
520
|
-
|
|
521
|
-
#### 1. Configuration file
|
|
522
|
-
```text
|
|
523
|
-
Refer to: evalscope/registry/config/cfg_pairwise_baseline.yaml
|
|
524
|
-
Parameters:
|
|
525
|
-
questions_file: question data path
|
|
526
|
-
answers_gen: candidate model prediction result generation, supports multiple models, can control whether to enable the model through the enable parameter
|
|
527
|
-
reviews_gen: evaluation result generation, currently defaults to using GPT-4 as the Auto-reviewer, can control whether to enable this step through the enable parameter
|
|
528
|
-
rating_gen: rating algorithm, can control whether to enable this step through the enable parameter, note that this step depends on the review_file must exist
|
|
529
|
-
```
|
|
530
|
-
#### 2. Execute the script
|
|
531
|
-
```shell
|
|
532
|
-
# Example:
|
|
533
|
-
python evalscope/run_arena.py --c registry/config/cfg_pairwise_baseline.yaml
|
|
534
|
-
```
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
## Datasets list
|
|
538
|
-
|
|
539
|
-
| DatasetName | Link | Status | Note |
|
|
540
|
-
|--------------------|----------------------------------------------------------------------------------------|--------|------|
|
|
541
|
-
| `mmlu` | [mmlu](https://modelscope.cn/datasets/modelscope/mmlu/summary) | Active | |
|
|
542
|
-
| `ceval` | [ceval](https://modelscope.cn/datasets/modelscope/ceval-exam/summary) | Active | |
|
|
543
|
-
| `gsm8k` | [gsm8k](https://modelscope.cn/datasets/modelscope/gsm8k/summary) | Active | |
|
|
544
|
-
| `arc` | [arc](https://modelscope.cn/datasets/modelscope/ai2_arc/summary) | Active | |
|
|
545
|
-
| `hellaswag` | [hellaswag](https://modelscope.cn/datasets/modelscope/hellaswag/summary) | Active | |
|
|
546
|
-
| `truthful_qa` | [truthful_qa](https://modelscope.cn/datasets/modelscope/truthful_qa/summary) | Active | |
|
|
547
|
-
| `competition_math` | [competition_math](https://modelscope.cn/datasets/modelscope/competition_math/summary) | Active | |
|
|
548
|
-
| `humaneval` | [humaneval](https://modelscope.cn/datasets/modelscope/humaneval/summary) | Active | |
|
|
549
|
-
| `bbh` | [bbh](https://modelscope.cn/datasets/modelscope/bbh/summary) | Active | |
|
|
550
|
-
| `race` | [race](https://modelscope.cn/datasets/modelscope/race/summary) | Active | |
|
|
551
|
-
| `trivia_qa` | [trivia_qa](https://modelscope.cn/datasets/modelscope/trivia_qa/summary) | To be intergrated | |
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
## Leaderboard
|
|
555
|
-
The LLM Leaderboard aims to provide an objective and comprehensive evaluation standard and platform to help researchers and developers understand and compare the performance of models on various tasks on ModelScope.
|
|
556
|
-
|
|
557
|
-
[Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
## Experiments and Results
|
|
562
|
-
[Experiments](./resources/experiments.md)
|
|
563
|
-
|
|
564
|
-
## Model Serving Performance Evaluation
|
|
565
|
-
[Perf](evalscope/perf/README.md)
|
|
566
|
-
|
|
567
|
-
## TO-DO List
|
|
568
|
-
- ✅Agents evaluation
|
|
569
|
-
- [ ] vLLM
|
|
570
|
-
- [ ] Distributed evaluating
|
|
571
|
-
- ✅ Multi-modal evaluation
|
|
572
|
-
- [ ] Benchmarks
|
|
573
|
-
- [ ] GAIA
|
|
574
|
-
- [ ] GPQA
|
|
575
|
-
- ✅ MBPP
|
|
576
|
-
- [ ] Auto-reviewer
|
|
577
|
-
- [ ] Qwen-max
|
|
578
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|