evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/wmt/wmt24_adapter.py

@@ -0,0 +1,294 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, ContentText
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT_TEMPLATE = """
+Translate the following {source_language} sentence into {target_language}:
+
+{source_language}: {source_text}
+{target_language}:
+""".strip()
+
+LANGUAGE_PAIRS = [
+    'en-ar_eg',
+    'en-ar_sa',
+    'en-bg_bg',
+    'en-bn_in',
+    'en-ca_es',
+    'en-cs_cz',
+    'en-da_dk',
+    'en-de_de',
+    'en-el_gr',
+    'en-es_mx',
+    'en-et_ee',
+    'en-fa_ir',
+    'en-fi_fi',
+    'en-fil_ph',
+    'en-fr_ca',
+    'en-fr_fr',
+    'en-gu_in',
+    'en-he_il',
+    'en-hi_in',
+    'en-hr_hr',
+    'en-hu_hu',
+    'en-id_id',
+    'en-is_is',
+    'en-it_it',
+    'en-ja_jp',
+    'en-kn_in',
+    'en-ko_kr',
+    'en-lt_lt',
+    'en-lv_lv',
+    'en-ml_in',
+    'en-mr_in',
+    'en-nl_nl',
+    'en-no_no',
+    'en-pa_in',
+    'en-pl_pl',
+    'en-pt_br',
+    'en-pt_pt',
+    'en-ro_ro',
+    'en-ru_ru',
+    'en-sk_sk',
+    'en-sl_si',
+    'en-sr_rs',
+    'en-sv_se',
+    'en-sw_ke',
+    'en-sw_tz',
+    'en-ta_in',
+    'en-te_in',
+    'en-th_th',
+    'en-tr_tr',
+    'en-uk_ua',
+    'en-ur_pk',
+    'en-vi_vn',
+    'en-zh_cn',
+    'en-zh_tw',
+    'en-zu_za',
+]
+
+LANGUAGE_BY_CODE = {
+    'ar_eg': 'arabic',
+    'ar_sa': 'arabic',
+    'bg_bg': 'bulgarian',
+    'bn_bd': 'bengali',
+    'bn_in': 'bengali',
+    'ca_es': 'catalan',
+    'cs_cz': 'czech',
+    'da_dk': 'danish',
+    'de_de': 'german',
+    'el_gr': 'greek',
+    'es_mx': 'spanish',
+    'et_ee': 'estonian',
+    'fa_ir': 'farsi',
+    'fi_fi': 'finnish',
+    'fil_ph': 'filipino',
+    'fr_ca': 'french',
+    'fr_fr': 'french',
+    'gu_in': 'gujarati',
+    'he_il': 'hebrew',
+    'hi_in': 'hindi',
+    'hr_hr': 'croatian',
+    'hu_hu': 'hungarian',
+    'id_id': 'indonesian',
+    'is_is': 'icelandic',
+    'it_it': 'italian',
+    'ja_jp': 'japanese',
+    'kn_in': 'kannada',
+    'ko_kr': 'korean',
+    'lt_lt': 'lithuanian',
+    'lv_lv': 'latvian',
+    'ml_in': 'malayalam',
+    'mr_in': 'marathi',
+    'nl_nl': 'dutch',
+    'no_no': 'norwegian',
+    'pa_in': 'punjabi',
+    'pl_pl': 'polish',
+    'pt_br': 'portuguese',
+    'pt_pt': 'portuguese',
+    'ro_ro': 'romanian',
+    'ru_ru': 'russian',
+    'sk_sk': 'slovak',
+    'sl_si': 'slovenian',
+    'sr_rs': 'serbian',
+    'sv_se': 'swedish',
+    'sw_ke': 'swahili',
+    'sw_tz': 'swahili',
+    'ta_in': 'tamil',
+    'te_in': 'telugu',
+    'th_th': 'thai',
+    'tr_tr': 'turkish',
+    'uk_ua': 'ukrainian',
+    'ur_pk': 'urdu',
+    'vi_vn': 'vietnamese',
+    'zh_cn': 'mandarin',
+    'zh_tw': 'mandarin',
+    'zu_za': 'zulu',
+    'en': 'english',
+}
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='wmt24pp',
+        pretty_name='WMT2024++',
+        dataset_id='extraordinarylab/wmt24pp',
+        tags=[Tags.MULTI_LINGUAL, Tags.MT],
+        description=(
+            'WMT2024 news translation benchmark supporting multiple language pairs. '
+            'Each subset represents a specific translation direction'
+        ),
+        subset_list=LANGUAGE_PAIRS,
+        eval_split='test',
+        metric_list={
+            'bleu': {},
+            'bert_score': {
+                'model_id_or_path': 'AI-ModelScope/xlm-roberta-large',
+                'model_type': 'xlm-roberta-large'
+            },
+            'comet': {
+                'model_id_or_path': 'evalscope/wmt22-comet-da',
+            }
+        },
+        few_shot_num=0,
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class WMT24PPAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize adapter and configure dataset subsets."""
+        super().__init__(**kwargs)
+        self.reformat_subset = True
+        self.use_batch_scoring = True  # Enable batch scoring
+
+        if 'comet' in self.metric_list:
+            check_import('comet', 'unbabel-comet', raise_error=True, feature_name='COMETScore Metric')
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+        """
+        source_text = str(record['source'])
+        target_text = str(record['target'])
+        language_pair = str(record['language_pair'])
+        source_language, target_language = language_pair.split('-')
+
+        # Format the generation prompt with the text
+        input_prompt = self.prompt_template.format(
+            source_text=source_text,
+            source_language=LANGUAGE_BY_CODE[source_language],
+            target_language=LANGUAGE_BY_CODE[target_language],
+        )
+
+        # Create content list for the input
+        content_list = [ContentText(text=input_prompt)]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=target_text,
+            subset_key=language_pair,
+            metadata={
+                'source_text': source_text,
+                'target_text': target_text,
+                'source_language': source_language,
+                'target_language': target_language,
+            },
+        )
+
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """Compute per-sample translation metrics."""
+        # Create a Score object for the current sample
+        score = Score(
+            prediction=original_prediction,
+            extracted_prediction=filtered_prediction,
+            value={},
+        )
+
+        # ---- BLEU ----
+        if 'bleu' in self.metric_list:
+            try:
+                from evalscope.metrics import bleu_ngram_one_sample
+
+                bleu_results = bleu_ngram_one_sample(filtered_prediction, reference)
+                score.value.update(bleu_results)
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] BLEU single-sample calculation failed: {e}')
+        return score
+
+    def batch_match_score(
+        self,
+        original_predictions: List[str],
+        filtered_predictions: List[str],
+        references: List[str],
+        task_states: List[TaskState],
+    ) -> List[Score]:
+        """Compute batched translation metrics (BLEU, BERTScore, COMET)."""
+        scores: List[Score] = []
+        for i in range(len(original_predictions)):
+            score = Score(
+                extracted_prediction=filtered_predictions[i],
+                prediction=original_predictions[i],
+                value={},
+            )
+            scores.append(score)
+
+        # ---- BLEU (per-sample within batch) ----
+        if 'bleu' in self.metric_list:
+            try:
+                from evalscope.metrics import bleu_ngram_one_sample
+
+                for i in range(len(scores)):
+                    bleu_results = bleu_ngram_one_sample(filtered_predictions[i], references[i])
+                    scores[i].value.update(bleu_results)
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] BLEU batch calculation failed: {e}')
+
+        # ---- BERTScore ----
+        if 'bert_score' in self.metric_list:
+            try:
+                from evalscope.metrics.metric import BertScore
+
+                score_args = self.metric_list.get('bert_score', {})
+                bert_scorer = BertScore(**score_args)
+                bert_score_f1 = bert_scorer.apply(filtered_predictions, references)
+                for i in range(len(scores)):
+                    scores[i].value.update({'bert_score': bert_score_f1[i]})
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] BERTScore batch calculation failed: {e}')
+
+        # ---- COMET ----
+        if 'comet' in self.metric_list:
+            try:
+                from evalscope.metrics.metric import COMETScore
+
+                score_args = self.metric_list.get('comet', {})
+                comet_scorer = COMETScore(**score_args)
+                data = [{
+                    'src': st.metadata.get('source_text'),
+                    'mt': pred,
+                    'ref': ref
+                } for pred, ref, st in zip(filtered_predictions, references, task_states)]
+                comet_scores = comet_scorer.apply(data)
+                for i in range(len(scores)):
+                    scores[i].value.update({'comet': comet_scores[i]})
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] COMET batch calculation failed: {e}')
+
+        return scores

evalscope/benchmarks/zerobench/zerobench_adapter.py

@@ -0,0 +1,64 @@
+# flake8: noqa: E501
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64, compress_image_to_limit
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# 定义提示模板
+PROMPT_TEMPLATE = """{question}
+\n\n\nLet's think step by step and give the final answer in curly braces,
+like this: {{final answer}}"
+"""
+
+SUBSET_LIST = ['default']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='zerobench',
+        pretty_name='ZeroBench',
+        dataset_id='evalscope/zerobench',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.MULTI_MODAL],
+        description=
+        'ZeroBench is a challenging visual reasoning benchmark for Large Multimodal Models (LMMs). It consists of a main set of 100 high-quality, manually curated questions covering numerous domains, reasoning types and image type. Questions in ZeroBench have been designed and calibrated to be beyond the capabilities of current frontier models. As such, none of the evaluated models achieves a non-zero pass@1 (with greedy decoding) or 5/5 reliability score.',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='zerobench',
+        train_split='zerobench_subquestions',
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class ZeroBenchAdapter(VisionLanguageAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question_text']
+        content_list: List[Content] = [ContentText(text=self.prompt_template.format(question=question))]
+        image = record['question_images_decoded']
+        if len(image) > 0:
+            for img in image:
+                # Ensure image is under OpenAI's 10MB data-URI limit by compressing if needed
+                processed_bytes, fmt = compress_image_to_limit(img['bytes'], 10_000_000)
+                image_base64 = bytes_to_base64(processed_bytes, format=fmt, add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        metadata = {
+            'question_id': record['question_id'],
+            'question_images': record['question_images'],
+            'image_attribution': record['image_attribution']
+        }
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)], target=record['question_answer'], metadata=metadata
+        )

evalscope/cli/cli.py

@@ -2,6 +2,7 @@
 
 import argparse
 
+from evalscope import __version__
 from evalscope.cli.start_app import StartAppCMD
 from evalscope.cli.start_eval import EvalCMD
 from evalscope.cli.start_perf import PerfBenchCMD
@@ -9,6 +10,7 @@ from evalscope.cli.start_perf import PerfBenchCMD
 
 def run_cmd():
     parser = argparse.ArgumentParser('EvalScope Command Line tool', usage='evalscope <command> [<args>]')
+    parser.add_argument('-v', '--version', action='version', version=f'evalscope {__version__}')
     subparsers = parser.add_subparsers(help='EvalScope command line helper.')
 
     PerfBenchCMD.define_args(subparsers)

evalscope/cli/start_app.py

@@ -3,7 +3,6 @@ import os
 from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
-from evalscope.report.app import create_app
 
 
 def subparser_func(args):
@@ -22,8 +21,19 @@ class StartAppCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.app import add_argument
+
         parser = parsers.add_parser(StartAppCMD.name)
+        add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        create_app()
+        try:
+            from evalscope.app import create_app
+        except ImportError as e:
+            raise ImportError(
+                f'Failed to import create_app from evalscope.app, due to {e}. '
+                "Please run `pip install 'evalscope[app]'`."
+            )
+
+        create_app(self.args)

evalscope/cli/start_eval.py

@@ -1,10 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os
 from argparse import ArgumentParser
 
-from evalscope.arguments import add_argument
 from evalscope.cli.base import CLICommand
-from evalscope.run import run_task
 
 
 def subparser_func(args):
@@ -23,9 +20,13 @@ class EvalCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.arguments import add_argument
+
         parser = parsers.add_parser(EvalCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
+        from evalscope.run import run_task
+
         run_task(self.args)

evalscope/cli/start_perf.py

@@ -3,8 +3,6 @@ import os
 from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
-from evalscope.perf.arguments import add_argument
-from evalscope.perf.main import run_perf_benchmark
 
 
 def subparser_func(args):
@@ -23,9 +21,19 @@ class PerfBenchCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.perf.arguments import add_argument
+
         parser = parsers.add_parser(PerfBenchCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
+        try:
+            from evalscope.perf.main import run_perf_benchmark
+        except ImportError as e:
+            raise ImportError(
+                f'Failed to import run_perf_benchmark from evalscope.perf.main, due to {e}. '
+                "Please run `pip install 'evalscope[perf]'`."
+            )
+
         run_perf_benchmark(self.args)

evalscope/cli/start_server.py

@@ -25,14 +25,16 @@ def add_perf_args(parser):
         '--logdir',
         required=True,
         type=str,
-        help='The monitor log save dir, tensorboard start at this path for display!')
+        help='The monitor log save dir, tensorboard start at this path for display!'
+    )
     parser.add_argument('--host', type=str, default='0.0.0.0', help='The tensorboard host')
     parser.add_argument('--tensorboard-port', type=str, default='6006', help='The tensorboard port')
 
 
 def async_run_command_with_popen(cmd):
     sub_process = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding='utf8')
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding='utf8'
+    )
     return sub_process
 
 
@@ -61,7 +63,8 @@ def start_server(args):
         bufsize=1,
         shell=True,
         universal_newlines=True,
-        encoding='utf8')
+        encoding='utf8'
+    )
 
     os.set_blocking(sub_process.stdout.fileno(), False)
     return sub_process

evalscope/collections/__init__.py

@@ -1,3 +1,27 @@
-from evalscope.collections.evaluator import EvaluatorCollection
-from evalscope.collections.sampler import StratifiedSampler, UniformSampler, WeightedSampler
-from evalscope.collections.schema import CollectionSchema, DatasetInfo
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .sampler import DatasetEntry, StratifiedSampler, UniformSampler, WeightedSampler
+    from .schema import CollectionSchema, DatasetInfo
+
+else:
+    _import_structure = {
+        'sampler': ['StratifiedSampler', 'UniformSampler', 'WeightedSampler', 'DatasetEntry'],
+        'schema': [
+            'CollectionSchema',
+            'DatasetInfo',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/collections/sampler.py

@@ -1,18 +1,17 @@
 import random
 from abc import ABC, abstractmethod
-from dataclasses import asdict, dataclass, field
+from pydantic import BaseModel, Field
 from tqdm import tqdm
 from typing import List, Optional
 
 from evalscope.collections.schema import CollectionSchema, DatasetInfo
 
 
-@dataclass
-class DatasetEntry:
+class DatasetEntry(BaseModel):
     index: int = 0
-    prompt: dict = field(default_factory=dict)
-    tags: List[str] = field(default_factory=list)
-    categories: List[str] = field(default_factory=list)
+    prompt: dict = Field(default_factory=dict)
+    tags: List[str] = Field(default_factory=list)
+    categories: List[str] = Field(default_factory=list)
     task_type: str = ''
     weight: float = 0.0
     dataset_name: str = ''
@@ -33,25 +32,27 @@ class Sampler(ABC):
         all_data = []
         data_dict = dataset.get_data()
         for subset_name, subset_data in data_dict.items():
-            for prompt in subset_data:
+            for sample in subset_data:
                 all_data.append(
                     DatasetEntry(
-                        prompt=prompt,
+                        prompt=sample.model_dump(exclude_none=True),
                         tags=dataset.tags,
                         categories=dataset.hierarchy,
                         task_type=dataset.task_type,
                         weight=dataset.weight,
                         dataset_name=dataset.name,
                         subset_name=subset_name,
-                    ))
-        sampled_data = random.choices(all_data, k=count)
+                    )
+                )
+        count = min(count, len(all_data))  # avoid sampling more than the dataset size
+        sampled_data = random.sample(all_data, k=count)
         return sampled_data
 
     def _update_index(self, all_data: List[DatasetEntry]) -> List[dict]:
         result = []
         for i, entry in enumerate(all_data):
             entry.index = i
-            result.append(asdict(entry))
+            result.append(entry.model_dump())
         return result
 
 

evalscope/collections/schema.py

@@ -3,6 +3,10 @@ import json
 from dataclasses import asdict, dataclass, field
 from typing import List, Union
 
+from evalscope.api.dataset import DatasetDict
+from evalscope.api.registry import get_benchmark
+from evalscope.config import TaskConfig
+
 
 @dataclass
 class DatasetInfo:
@@ -13,16 +17,11 @@ class DatasetInfo:
     args: dict = field(default_factory=dict)
     hierarchy: List[str] = field(default_factory=list)
 
-    def get_data(self) -> dict:
-        from evalscope.benchmarks import Benchmark
-
-        benchmark_meta = Benchmark.get(self.name)
-
-        data_adapter = benchmark_meta.get_data_adapter(config=self.args)
-        data_dict = data_adapter.load(
-            dataset_name_or_path=benchmark_meta.dataset_id, subset_list=benchmark_meta.subset_list)
-        prompts = data_adapter.gen_prompts(data_dict)
-        return prompts
+    def get_data(self) -> DatasetDict:
+        dataset_args = {self.name: self.args}
+        benchmark_meta = get_benchmark(self.name, config=TaskConfig(dataset_args=dataset_args))
+        data_dict = benchmark_meta.load_dataset()
+        return data_dict
 
 
 def flatten_weight(collection: 'CollectionSchema', base_weight=1):
@@ -112,8 +111,10 @@ if __name__ == '__main__':
             ]),
             CollectionSchema(
                 name='chinese',
-                datasets=[DatasetInfo(name='ceval', weight=1, tags=['zh'], args={'subset_list': ['logic']})])
-        ])
+                datasets=[DatasetInfo(name='ceval', weight=1, tags=['zh'], args={'subset_list': ['logic']})]
+            )
+        ]
+    )
     print(schema)
     print(schema.flatten())
     schema.dump_json('outputs/schema.json')