evalscope 0.9.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

evalscope/collections/evaluator.py

@@ -4,13 +4,15 @@ import pandas as pd
 from collections import defaultdict
 from tabulate import tabulate
 from tqdm import tqdm
+from typing import List
 
 from evalscope.benchmarks import Benchmark
 from evalscope.collections.sampler import DatasetEntry
 from evalscope.config import TaskConfig
-from evalscope.constants import AnswerKeys, DumpMode, EvalType, ReviewKeys
+from evalscope.constants import DataCollection, DumpMode
 from evalscope.evaluator import Evaluator
 from evalscope.models import get_local_model, initialize_model_adapter
+from evalscope.report import ReportGenerator
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -38,6 +40,12 @@ class SimpleEvaluator(Evaluator):
         review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
         return review_d
 
+    def get_score(self, review_d) -> float:
+        metric_score: List[dict] = self.compute_metrics(reviews_list=[review_d])
+        # use the first metric by default
+        score = metric_score[0]['score']
+        return score
+
 
 class EvaluatorCollection:
 
@@ -45,16 +53,18 @@ class EvaluatorCollection:
         self.task_cfg = task_cfg
         self.outputs = outputs
         self.model = get_local_model(task_cfg)
-        self.dataset = self.load()
+        self.dataset, self.dataset_name = self.load()
         self.dataset_name_map, self.dataset_id_map = self._parse_dataset()
         self.evaluators = self._initialize_evaluators()
 
-    def load(self) -> list[DatasetEntry]:
-        raw_dataset = jsonl_to_list(self.task_cfg.dataset_args['data_collection']['local_path'])
+    def load(self) -> tuple[list[DatasetEntry], str]:
+        dataset_path = self.task_cfg.dataset_args[DataCollection.NAME]['local_path']
+        dataset_name = os.path.basename(dataset_path).split('.')[0]
+        raw_dataset = jsonl_to_list(dataset_path)
         datasets = []
         for sample in raw_dataset:
             datasets.append(DatasetEntry(**sample))
-        return datasets
+        return datasets, dataset_name
 
     def _parse_dataset(self):
         dataset_name_map = defaultdict(lambda: defaultdict(list))
@@ -75,65 +85,80 @@ class EvaluatorCollection:
                                                        self.outputs)
         return evaluators
 
-    def get_report(self, reviews):
-        data = []
-        for dataset_name, data_map in self.dataset_name_map.items():
-            for subset_name, ids in data_map.items():
-                for _id in ids:
-                    review_d = reviews[_id]
-                    row_data: DatasetEntry = self.dataset_id_map[_id]
-                    score = self.get_pred_score(review_d)
-                    data.append({
-                        'task_type': row_data.task,
-                        'dataset_name': dataset_name,
-                        'subset_name': subset_name,
-                        'tags': row_data.tags,
-                        'score': score
-                    })
-
-        df = pd.DataFrame(data)
-        # Explode tags to multiple rows
-        df_exploded = df.explode('tags')
-
-        # Helper function for aggregation and sorting
+    def get_report(self, scores):
+
+        def get_dataframe(scores):
+            data = []
+            for dataset_name, data_map in self.dataset_name_map.items():
+                for subset_name, ids in data_map.items():
+                    for _id in ids:
+                        row_data: DatasetEntry = self.dataset_id_map[_id]
+                        score = scores[_id]
+                        data.append(
+                            dict(
+                                task_type=row_data.task_type,
+                                categories=tuple(row_data.categories),
+                                dataset_name=dataset_name,
+                                subset_name=subset_name,
+                                tags=row_data.tags,
+                                score=score))
+            return pd.DataFrame(data)
+
         def aggregate_and_sort(df, group_by_cols):
+            # aggregate by group_by_cols, and calculate average_score and count
             report_df = df.groupby(group_by_cols) \
                 .agg(average_score=('score', 'mean'), count=('score', 'size')) \
                 .reset_index()
-
-            # Round average_score to 4 decimal places
             report_df['average_score'] = report_df['average_score'].round(4)
-
             report_df = report_df.sort_values(by='count', ascending=False) \
                 .to_dict(orient='records')
             return report_df
 
-        # Multi-level aggregation
+        df = get_dataframe(scores)
+
+        # multi-level aggregation
         subset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name', 'subset_name'])
         dataset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name'])
         task_report_df = aggregate_and_sort(df, ['task_type'])
-        tag_report_df = aggregate_and_sort(df_exploded, ['tags'])
 
-        # Convert sorted DataFrames to Dict
-        report = {
+        # explode tags to multiple rows
+        df_exploded_tags = df.explode('tags')
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags'])
+
+        # process multi-level categories
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories['categories'].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'category{level}'] = df_categories['categories'].apply(lambda x: x[level]
+                                                                                  if len(x) > level else '')
+        category_report_df = aggregate_and_sort(df_categories, [f'category{level}' for level in range(max_depth)])
+
+        # convert to dict format
+        report_dict = {
             'subset_level': subset_report_df,
             'dataset_level': dataset_report_df,
             'task_level': task_report_df,
-            'tag_level': tag_report_df
+            'tag_level': tag_report_df,
+            'category_level': category_report_df,
         }
 
-        # Log the report
-        for level, data in report.items():
+        # record report
+        for level, data in report_dict.items():
             table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
             logger.info(f'{level} Report:\n{table}')
 
-        # Save the report to a JSON file
-        report_file_path = os.path.join(self.outputs.reports_dir, 'data_collection.json')
+        report = ReportGenerator.gen_collection_report(df, self.dataset_name, self.task_cfg.model_id)
+        # save report to JSON file
+        report_file_path = os.path.join(self.outputs.reports_dir, self.task_cfg.model_id, f'{self.dataset_name}.json')
+        os.makedirs(os.path.dirname(report_file_path), exist_ok=True)
         with open(report_file_path, 'w', encoding='utf-8') as f:
-            json.dump(report, f, ensure_ascii=False, indent=4)
+            json.dump(report.to_dict(), f, ensure_ascii=False, indent=4)
 
     def get_answers(self):
-        pred_file_path = os.path.join(self.outputs.predictions_dir, 'data_collection.jsonl')
+        pred_file_path = os.path.join(self.outputs.predictions_dir, self.task_cfg.model_id,
+                                      f'{self.dataset_name}.jsonl')
+        os.makedirs(os.path.dirname(pred_file_path), exist_ok=True)
         answers = defaultdict(dict)
         for sample in tqdm(self.dataset, desc='Getting answers'):
             evaluator = self.evaluators[sample.dataset_name]
@@ -143,36 +168,31 @@ class EvaluatorCollection:
         return answers
 
     def get_reviews(self, answers):
-        review_file_path = os.path.join(self.outputs.reviews_dir, 'data_collection.jsonl')
+        review_file_path = os.path.join(self.outputs.reviews_dir, self.task_cfg.model_id)
+        os.makedirs(review_file_path, exist_ok=True)
         reviews = defaultdict(dict)
         for sample in tqdm(self.dataset, desc='Getting reviews'):
             evaluator = self.evaluators[sample.dataset_name]
             review_d = evaluator.get_review(answers[sample.index])
             reviews[sample.index] = review_d
-            dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
+            dump_jsonl_data(
+                review_d,
+                os.path.join(review_file_path, f'{self.dataset_name}_{sample.dataset_name}_{sample.subset_name}.jsonl'),
+                dump_mode=DumpMode.APPEND)
         return reviews
 
-    @staticmethod
-    def get_pred_score(review_d) -> float:
-        return float(review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT])
+    def get_scores(self, reviews) -> float:
+        scores = defaultdict(dict)
+        for sample in tqdm(self.dataset, desc='Getting scores'):
+            evaluator = self.evaluators[sample.dataset_name]
+            review_d = reviews[sample.index]
+            score = evaluator.get_score(review_d)
+            scores[sample.index] = score
+
+        return scores
 
     def eval(self, **kwargs):
         answers = self.get_answers()
         reviews = self.get_reviews(answers)
-        self.get_report(reviews)
-
-
-if __name__ == '__main__':
-    task_cfg = TaskConfig(
-        model='qwen2.5',
-        api_url='http://127.0.0.1:8801/v1/chat/completions',
-        api_key='EMPTY',
-        eval_type=EvalType.SERVICE,
-        datasets=['data_collection'],
-        dataset_args={'data_collection': {
-            'local_path': 'outputs/mixed_data.jsonl'
-        }},
-    )
-
-    evaluator_collection = EvaluatorCollection(task_cfg)
-    evaluator_collection.eval()
+        scores = self.get_scores(reviews)
+        self.get_report(scores)

evalscope/collections/sampler.py

@@ -12,7 +12,8 @@ class DatasetEntry:
     index: int = 0
     prompt: dict = field(default_factory=dict)
     tags: List[str] = field(default_factory=list)
-    task: str = ''
+    categories: List[str] = field(default_factory=list)
+    task_type: str = ''
     weight: float = 0.0
     dataset_name: str = ''
     subset_name: str = ''
@@ -21,30 +22,30 @@ class DatasetEntry:
 # Define an abstract base class for Samplers
 class Sampler(ABC):
 
-    def __init__(self, schema: CollectionSchema, count: Optional[int] = None):
+    def __init__(self, schema: CollectionSchema):
         self.schema = schema
-        self.count = count
 
     @abstractmethod
     def sample(self) -> List[dict]:
         raise NotImplementedError
 
-    def _collect_dataset_data(self, dataset_info_list: List[DatasetInfo]) -> List[DatasetEntry]:
+    def _sample_dataset(self, dataset: DatasetInfo, count: int) -> List[DatasetEntry]:
         all_data = []
-        for dataset in tqdm(dataset_info_list, desc='Collecting dataset data'):
-            data_dict = dataset.get_data()
-            for subset_name, subset_data in data_dict.items():
-                for prompt in subset_data:
-                    all_data.append(
-                        DatasetEntry(
-                            prompt=prompt,
-                            tags=dataset.tags,
-                            task=dataset.task_type,
-                            weight=dataset.weight,
-                            dataset_name=dataset.name,
-                            subset_name=subset_name,
-                        ))
-        return all_data
+        data_dict = dataset.get_data()
+        for subset_name, subset_data in data_dict.items():
+            for prompt in subset_data:
+                all_data.append(
+                    DatasetEntry(
+                        prompt=prompt,
+                        tags=dataset.tags,
+                        categories=dataset.hierarchy,
+                        task_type=dataset.task_type,
+                        weight=dataset.weight,
+                        dataset_name=dataset.name,
+                        subset_name=subset_name,
+                    ))
+        sampled_data = random.choices(all_data, k=count)
+        return sampled_data
 
     def _update_index(self, all_data: List[DatasetEntry]) -> List[dict]:
         result = []
@@ -59,21 +60,19 @@ class WeightedSampler(Sampler):
     Weighted sampler, according to the weight of each dataset, sample data from each dataset.
     """
 
-    def sample(self) -> List[dict]:
+    def sample(self, count: int) -> List[dict]:
         dataset_info_list = self.schema.flatten()
-        all_data = self._collect_dataset_data(dataset_info_list)
-
-        remaining_count = self.count
         sampled_data = []
+        remaining_count = count
 
         for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
             if i == len(dataset_info_list) - 1:
                 dataset_sample_count = remaining_count
             else:
-                dataset_sample_count = int(dataset.weight * self.count)
+                dataset_sample_count = int(dataset.weight * count)
                 remaining_count -= dataset_sample_count
 
-            sampled_data.extend(random.choices(all_data, k=dataset_sample_count))
+            sampled_data.extend(self._sample_dataset(dataset, dataset_sample_count))
 
         return self._update_index(sampled_data)
 
@@ -83,16 +82,20 @@ class UniformSampler(Sampler):
     Uniform sampler, sample data from each dataset with the same number of samples.
     """
 
-    def sample(self) -> List[dict]:
+    def sample(self, count: int) -> List[dict]:
         dataset_info_list = self.schema.flatten()
-        all_data = self._collect_dataset_data(dataset_info_list)
-
         num_datasets = len(dataset_info_list)
-        samples_per_dataset = self.count // num_datasets
+        remaining_count = count
         sampled_data = []
 
-        for _ in tqdm(dataset_info_list, desc='Sampling data'):
-            sampled_data.extend(random.choices(all_data, k=samples_per_dataset))
+        for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
+            if i == len(dataset_info_list) - 1:
+                dataset_sample_count = remaining_count
+            else:
+                dataset_sample_count = count // num_datasets
+                remaining_count -= dataset_sample_count
+
+            sampled_data.extend(self._sample_dataset(dataset, dataset_sample_count))
 
         return self._update_index(sampled_data)
 
@@ -102,18 +105,21 @@ class StratifiedSampler(Sampler):
     Stratified sampler, sample data from each dataset according to the number of samples of each dataset.
     """
 
-    def sample(self) -> List[dict]:
+    def sample(self, count: int) -> List[dict]:
         dataset_info_list = self.schema.flatten()
-        all_data = self._collect_dataset_data(dataset_info_list)
 
         total_samples = sum(len(dataset.get_data()) for dataset in dataset_info_list)
+        remaining_count = count
         sampled_data = []
 
-        for dataset in tqdm(dataset_info_list, desc='Sampling data'):
-            dataset_samples = len(dataset.get_data())
-            samples_for_dataset = int((dataset_samples / total_samples) * self.count)
-            sampled_data.extend(random.choices(all_data, k=samples_for_dataset))
+        for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
+            if i == len(dataset_info_list) - 1:
+                dataset_sample_count = remaining_count
+            else:
+                dataset_sample_count = int((len(dataset.get_data()) / total_samples) * count)
+                remaining_count -= dataset_sample_count
 
+            sampled_data.extend(self._sample_dataset(dataset, dataset_sample_count))
         return self._update_index(sampled_data)
 
 
@@ -122,11 +128,11 @@ if __name__ == '__main__':
 
     schema = CollectionSchema.from_json('outputs/schema.json')
     print(schema.to_dict())
-    mixed_data = WeightedSampler(schema, 100).sample()
+    mixed_data = WeightedSampler(schema).sample(10)
     dump_jsonl_data(mixed_data, 'outputs/weighted_mixed_data.jsonl')
 
-    mixed_data = UniformSampler(schema, 100).sample()
-    dump_jsonl_data(mixed_data, 'outputs/uniform_mixed_data.jsonl')
+    # mixed_data = UniformSampler(schema, 100).sample()
+    # dump_jsonl_data(mixed_data, 'outputs/uniform_mixed_data.jsonl')
 
-    mixed_data = StratifiedSampler(schema, 100).sample()
-    dump_jsonl_data(mixed_data, 'outputs/stratified_mixed_data.jsonl')
+    # mixed_data = StratifiedSampler(schema, 100).sample()
+    # dump_jsonl_data(mixed_data, 'outputs/stratified_mixed_data.jsonl')

evalscope/collections/schema.py

@@ -11,6 +11,7 @@ class DatasetInfo:
     task_type: str = ''
     tags: List[str] = field(default_factory=list)
     args: dict = field(default_factory=dict)
+    hierarchy: List[str] = field(default_factory=list)
 
     def get_data(self) -> dict:
         from evalscope.benchmarks import Benchmark
@@ -34,18 +35,15 @@ def flatten_weight(collection: 'CollectionSchema', base_weight=1):
             dataset.weight = current_weight
 
 
-def flatten_tags(collection: 'CollectionSchema', parent_names=None):
+def flatten_name(collection: 'CollectionSchema', parent_names=None):
     if parent_names is None:
         parent_names = []
     current_names = parent_names + [collection.name]
     for dataset in collection.datasets:
         if isinstance(dataset, CollectionSchema):
-            flatten_tags(dataset, current_names)
+            flatten_name(dataset, current_names)
         else:
-            # Add all parent CollectionSchema names to the tags of each DatasetInfo
-            for name in current_names:
-                if name not in dataset.tags:
-                    dataset.tags.append(name)
+            dataset.hierarchy = current_names.copy()
 
 
 def flatten_datasets(collection: 'CollectionSchema') -> List[DatasetInfo]:
@@ -100,7 +98,7 @@ class CollectionSchema:
 
     def flatten(self) -> List[DatasetInfo]:
         collection = copy.deepcopy(self)
-        flatten_tags(collection)
+        flatten_name(collection)
         flatten_weight(collection)
         return flatten_datasets(collection)
 
@@ -109,8 +107,12 @@ if __name__ == '__main__':
     schema = CollectionSchema(
         name='reasoning',
         datasets=[
-            DatasetInfo(name='arc', weight=1, task_type='reasoning', tags=['en']),
-            DatasetInfo(name='ceval', weight=1, task_type='reasoning', tags=['zh'], args={'subset_list': ['logic']})
+            CollectionSchema(name='english', datasets=[
+                DatasetInfo(name='arc', weight=1, tags=['en']),
+            ]),
+            CollectionSchema(
+                name='chinese',
+                datasets=[DatasetInfo(name='ceval', weight=1, tags=['zh'], args={'subset_list': ['logic']})])
         ])
     print(schema)
     print(schema.flatten())
@@ -118,5 +120,7 @@ if __name__ == '__main__':
 
     schema = CollectionSchema.from_json('outputs/schema.json')
     print(schema)
+    # 打印扁平化后的结果
     for dataset in schema.flatten():
-        print(dataset)
+        print(f'Dataset: {dataset.name}')
+        print(f"Hierarchy: {' -> '.join(dataset.hierarchy)}")

evalscope/constants.py

@@ -145,3 +145,7 @@ class EvalBackend:
     VLM_EVAL_KIT = 'VLMEvalKit'
     RAG_EVAL = 'RAGEval'
     THIRD_PARTY = 'ThirdParty'
+
+
+class DataCollection:
+    NAME = 'data_collection'

evalscope/evaluator/evaluator.py

@@ -12,7 +12,7 @@ from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
 from evalscope.constants import AnswerKeys, DumpMode, EvalStage, ReviewKeys
 from evalscope.models import BaseModelAdapter, CustomModelAdapter
-from evalscope.tools.combine_reports import gen_table
+from evalscope.report import Report, gen_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -43,8 +43,8 @@ class Evaluator(object):
                  task_cfg: TaskConfig = None,
                  **kwargs):
 
+        self.dataset_name = data_adapter.name
         self.dataset_name_or_path = os.path.expanduser(dataset_name_or_path)
-        self.dataset_name = os.path.basename(self.dataset_name_or_path.rstrip(os.sep)).split('.')[0]
         self.model_name = task_cfg.model_id
         self.custom_task_name = f'{self.model_name}_{self.dataset_name}'
 
@@ -201,7 +201,7 @@ class Evaluator(object):
         # Gen review_id (concat: answer_id + reviewer_spec)
         answer_id = answer_d[AnswerKeys.ANSWER_ID]
         reviewer_spec = {
-            'metric': [metric_d['name'] for metric_d in self.data_adapter.metric_list],
+            'metric': [metric.name for metric in self.data_adapter.metric_list],
             'reviewer': ['Evaluator'],
             'revision': ['default']
         }
@@ -246,7 +246,7 @@ class Evaluator(object):
 
         return reviews_list
 
-    def compute_metrics(self, reviews_list: List[dict]) -> Any:
+    def compute_metrics(self, reviews_list: List[dict]) -> List[dict]:
         """
         To compute metrics from reviews_list for each subset.
         It is required to rewrite this method to support your own evaluator.
@@ -264,28 +264,37 @@ class Evaluator(object):
                 logger.warning(f'Review not finished for answer_id: {review_d[AnswerKeys.ANSWER_ID]}')
                 continue
 
-            review_res = review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT]
+            if len(review_d[AnswerKeys.CHOICES]) == 0:
+                logger.warning(f'No choices found for answer_id: {review_d[AnswerKeys.ANSWER_ID]}')
+                continue
+            elif len(review_d[AnswerKeys.CHOICES]) == 1:
+                review_res = review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT]
+            else:
+                review_res = [choice[ReviewKeys.REVIEW][ReviewKeys.RESULT] for choice in review_d[AnswerKeys.CHOICES]]
+
             review_res_list.append(review_res)
 
-        metric_score: Union[float, dict] = self.data_adapter.compute_metric(review_res_list=review_res_list)
+        metric_score: List[dict] = self.data_adapter.compute_metric(review_res_list=review_res_list)
 
         return metric_score
 
-    def dump_report(self, reviews_score_all: dict, use_table: bool = True):
+    def dump_report(self, reviews_score_all: List[dict], use_table: bool = True):
         """
         Get report for total reviews of specific dataset.
         It is required to rewrite this method to support your own evaluator.
 
         Args:
-            report_map: report dict. Generated by func self.data_adapter.gen_report().
+            reviews_score_all: reviews score list. Generated by func self.data_adapter.compute_metric().
             use_table: whether to generate table for reports. Default to True.
 
         Returns: None
         """
         # Get report map
-        report_map: dict = self.data_adapter.gen_report(
-            subset_score_map=reviews_score_all, report_name=self.custom_task_name)
-        report_map.update(dict(model_name=self.model_name, dataset_name=self.dataset_name))
+        report_map: Report = self.data_adapter.gen_report(
+            subset_score_map=reviews_score_all,
+            report_name=self.custom_task_name,
+            model_name=self.model_name,
+            dataset_name=self.dataset_name)
 
         # Dump report
         report_path: str = os.path.join(self.outputs_structure.reports_dir, self.model_name,
@@ -294,7 +303,7 @@ class Evaluator(object):
 
         # Write report
         with open(report_path, 'w') as f:
-            f.write(json.dumps(report_map, ensure_ascii=False, indent=4))
+            f.write(json.dumps(report_map.to_dict(), ensure_ascii=False, indent=4))
         logger.info(f'Dump report: {report_path} \n')
 
         # Make table
@@ -351,7 +360,7 @@ class Evaluator(object):
                 subset_name=subset_name, answers_list=answers_list, debug=debug, **kwargs)
 
             metric_res = self.compute_metrics(reviews_list=reviews_list)
-            reviews_score_all[subset_name] = (metric_res, len(reviews_list))
+            reviews_score_all[subset_name] = metric_res
             stage_reviews_dict[subset_name] = reviews_list
 
         if self.stage == EvalStage.INFER:

evalscope/metrics/__init__.py

@@ -1,7 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from evalscope.metrics.metrics import bleu_ngram_one_sample, exact_match, weighted_mean
+from evalscope.metrics.metrics import bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, weighted_mean
+from evalscope.metrics.named_metrics import *
 from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
-
-WeightedAverageAccuracy = {'name': 'WeightedAverageAccuracy', 'object': weighted_mean}
-WeightedAverageBLEU = {'name': 'WeightedAverageBLEU', 'object': weighted_mean}
-Pass1 = {'name': 'Pass@1', 'object': weighted_mean}

evalscope/metrics/metrics.py

@@ -1,6 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI. and its affiliates.
 # Copyright (c) OpenAI. and its affiliates.
+
 import itertools
 import math
 import numpy as np
@@ -8,7 +9,7 @@ import random
 import sacrebleu
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import Dict, List, Union
+from typing import TYPE_CHECKING, Dict, List, Union
 
 
 def mean(arr):
@@ -103,12 +104,20 @@ def perplexity(items):
     return math.exp(-mean(items))
 
 
-def weighted_mean(items) -> float:
+def weighted_mean(items: List) -> float:
     # e.g. [(0,1), (0.5,1), (1,1)]
     a, b = zip(*items)
     return sum(a) / sum(b)
 
 
+def micro_mean(items):
+    return sum([item.score * item.num for item in items]) / sum([item.num for item in items])
+
+
+def macro_mean(items):
+    return sum([item.score for item in items]) / len(items)
+
+
 def weighted_perplexity(items):
     return math.exp(-weighted_mean(items))
 

evalscope/metrics/named_metrics.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass, field
+from typing import Callable
+
+from evalscope.metrics.metrics import mean, weighted_mean
+
+
+@dataclass
+class Metric:
+    name: str = 'default_metric'
+    object: Callable = field(default_factory=lambda: mean)
+
+
+AverageAccuracy = Metric(name='AverageAccuracy', object=mean)
+WeightedAverageAccuracy = Metric(name='WeightedAverageAccuracy', object=weighted_mean)
+AverageBLEU = Metric(name='AverageBLEU', object=mean)
+WeightedAverageBLEU = Metric(name='WeightedAverageBLEU', object=weighted_mean)
+Pass1 = Metric(name='Pass@1', object=mean)

evalscope/models/chat_adapter.py

@@ -76,6 +76,8 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
             # For base model, use the query as the input
             formatted_prompt = query
 
+        logger.debug(f'formatted_prompt: {formatted_prompt}')
+
         inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
         input_ids = inputs['input_ids']
 

evalscope/models/server_adapter.py

@@ -46,9 +46,13 @@ class ServerModelAdapter(BaseModelAdapter):
             query = inputs
             system_prompt = None
         elif isinstance(inputs, dict):
-            # TODO: to be supported for continuation list like truthful_qa
-            query = inputs['data'][0]
-            system_prompt = inputs.get('system_prompt', None)
+            data: list = inputs['data']
+            if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+                query = '\n'.join(''.join(item) for item in data)
+                system_prompt = inputs.get('system_prompt', None)
+            else:
+                query = data[0]
+                system_prompt = inputs.get('system_prompt', None)
         elif isinstance(inputs, list):
             query = '\n'.join(inputs)
             system_prompt = None
@@ -76,10 +80,13 @@ class ServerModelAdapter(BaseModelAdapter):
     def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
         """Make request to remote API."""
         # Format request JSON according to OpenAI API format
+        do_sample = infer_cfg.get('do_sample', False)
+        temperature = infer_cfg.get('temperature', 0.0) if do_sample else 0.0
+
         request_json = {
             **content, 'model': self.model_id,
             'max_tokens': infer_cfg.get('max_tokens', 2048),
-            'temperature': infer_cfg.get('temperature', 0.0),
+            'temperature': temperature,
             'top_p': infer_cfg.get('top_p', 1.0),
             'n': infer_cfg.get('num_return_sequences', 1),
             'stop': infer_cfg.get('stop', None)

evalscope/perf/__init__.py

@@ -0,0 +1 @@
+from evalscope.perf.main import run_perf_benchmark