evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/backend/rag_eval/utils/llm.py

@@ -2,11 +2,10 @@ import os
 from langchain_core.callbacks.manager import CallbackManagerForLLMRun
 from langchain_core.language_models.llms import LLM as BaseLLM
 from langchain_openai import ChatOpenAI
-from transformers.generation.configuration_utils import GenerationConfig
 from typing import Any, Dict, Iterator, List, Mapping, Optional
 
-from evalscope.constants import DEFAULT_MODEL_REVISION
-from evalscope.models import ChatGenerationModelAdapter, LocalModel
+from evalscope.api.model import GenerateConfig, Model, get_model
+from evalscope.constants import DEFAULT_MODEL_REVISION, EvalType
 
 
 class LLM:
@@ -30,16 +29,19 @@ class LocalLLM(BaseLLM):
     model_name_or_path: str
     model_revision: str = DEFAULT_MODEL_REVISION
     template_type: Optional[str] = None
-    model_name: Optional[str]
-    model: Optional[ChatGenerationModelAdapter]
-    generation_config: Optional[Dict]
+    model_name: Optional[str] = None
+    model: Optional[Model] = None
+    generation_config: Optional[Dict] = {}
 
     def __init__(self, **kw):
         super().__init__(**kw)
         self.model_name = os.path.basename(self.model_name_or_path)
-        self.model = ChatGenerationModelAdapter(
-            model=LocalModel(model_id=self.model_name_or_path, model_revision=self.model_revision),
-            generation_config=GenerationConfig(**self.generation_config) if self.generation_config else None,
+
+        # Create and initialize the local model
+        self.model = get_model(
+            model=self.model_name_or_path,
+            eval_type=EvalType.CHECKPOINT,
+            config=GenerateConfig(**self.generation_config),
         )
 
     def _call(
@@ -50,10 +52,9 @@ class LocalLLM(BaseLLM):
         **kwargs: Any,
     ) -> str:
         """Run the LLM on the given input."""
-        infer_cfg = {'stop': stop}
 
-        response, _ = self.model.predict([{'data': [prompt]}], infer_cfg=infer_cfg)
-        return response[0][0]
+        response = self.model.generate(input=prompt)
+        return response.completion
 
     @property
     def _identifying_params(self) -> Dict[str, Any]:

evalscope/benchmarks/__init__.py

@@ -4,8 +4,6 @@ import importlib
 import os
 import time
 
-from evalscope.benchmarks.benchmark import Benchmark, BenchmarkMeta
-from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.utils import get_logger
 
 logger = get_logger()

evalscope/benchmarks/aigc/i2i/general_i2i_adapter.py

@@ -0,0 +1,44 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class GeneralI2IAdapter:
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
+        dataset_name_or_path = dataset_name_or_path or self.dataset_id
+        subset_list = subset_list or self.subset_list
+
+        data_file_dict = defaultdict(str)
+        data_item_dict = defaultdict(list)
+
+        # get data file path and subset name
+        if os.path.isdir(dataset_name_or_path):
+            for subset_name in subset_list:
+                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
+        elif os.path.isfile(dataset_name_or_path):
+            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
+            data_file_dict[cur_subset_name] = dataset_name_or_path
+        else:
+            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
+
+        # load data from local disk
+        try:
+            for subset_name, file_path in data_file_dict.items():
+                data_item_dict[subset_name] = jsonl_to_list(file_path)
+        except Exception as e:
+            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
+
+        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
+
+        return data_dict

evalscope/benchmarks/aigc/t2i/evalmuse_adapter.py

@@ -1,78 +1,76 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
 from collections import defaultdict
 from typing import List, Optional, Union
 
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.metric.scorer import AggScore, Score
+from evalscope.api.registry import get_metric, register_benchmark
+from evalscope.constants import Tags
 from evalscope.metrics import mean
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.function_utils import thread_safe
 from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='evalmuse',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['EvalMuse'],
-    metric_list=['FGA_BLIP2Score'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='evalmuse',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='EvalMuse Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['EvalMuse'],
+        metric_list=['FGA_BLIP2Score'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
 )
-class EvalMuseAdapter(T2IBaseAdapter):
+class EvalMuseAdapter(Text2ImageAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        assert len(self.metric_list
+                   ) == 1 and self.metric_list[0] == 'FGA_BLIP2Score', 'Only FGA_BLIP2Score is supported for EvalMuse'
 
-    def load(self, **kwargs) -> dict:
-        if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)
+    @thread_safe
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state):
+        # Get prediction and prompt from task state
+        image_path = task_state.metadata.get('image_path', original_prediction)
 
-    def get_gold_answer(self, input_d: dict) -> dict:
-        # return prompt and elements dict
-        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=image_path,
+            prediction=image_path,
+        )
 
-    def match(self, gold: dict, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            if metric_name == 'FGA_BLIP2Score':
-                # For FGA_BLIP2Score, we need to pass the dictionary
-                score = metric_func(images=[pred], texts=[gold])[0][0]
-            else:
-                score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
-            if isinstance(score, dict):
-                for k, v in score.items():
-                    res[f'{metric_name}:{k}'] = v.cpu().item()
-            else:
-                res[metric_name] = score.cpu().item()
-        return res
+        # Calculate scores for each configured metric
+        try:
+            metric_name = self.metric_list[0]
+            metric_cls = get_metric(metric_name)
+            metric_func = metric_cls()  # Initialize with parameters
+            metric_score = metric_func(image_path, task_state.metadata)[0]
 
-    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-        """
-        items = super().compute_dict_metric(review_res_list, **kwargs)
-        # add statistics for each metric
+            for k, v in metric_score.items():
+                score.value[f'{metric_name}:{k}'] = v.cpu().item()
+        except Exception as e:
+            logger.error(f'Error calculating metric {metric_name}: {e}')
+            score.value[metric_name] = 0
+            score.metadata[metric_name] = f'error: {str(e)}'
+
+        return score
+
+    def aggregate_scores(self, sample_scores) -> List[AggScore]:
         new_items = defaultdict(list)
-        for metric_name, value_list in items.items():
-            if 'FGA_BLIP2Score' in metric_name and '(' in metric_name:  # FGA_BLIP2Score element score
+        agg_list = []
+        for sample_score in sample_scores:
+            for metric_name, value in sample_score.score.value.items():
                 metrics_prefix = metric_name.split(':')[0]
                 category = metric_name.rpartition('(')[-1].split(')')[0]
                 category = category.split('-')[0].lower()  # remove the suffix if exists
-                new_items[f'{metrics_prefix}:{category}'].extend(value_list)
-            else:
-                new_items[metric_name].extend(value_list)
+                new_items[f'{metrics_prefix}:{category}'].append(value)
+
+        for k, v in new_items.items():
+            agg_list.append(AggScore(metric_name=k, score=mean(v), num=len(v)))
 
-        # calculate mean for each metric
-        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in new_items.items()]
+        return agg_list

evalscope/benchmarks/aigc/t2i/genai_bench_adapter.py

@@ -1,58 +1,53 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import get_metric, register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='genai_bench',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['GenAI-Bench-1600'],
-    metric_list=['VQAScore'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='genai_bench',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='GenAI-Bench Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['GenAI-Bench-1600'],
+        metric_list=['VQAScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
 )
-class GenAIBenchAdapter(T2IBaseAdapter):
+class GenAIBenchAdapter(Text2ImageAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def load(self, **kwargs) -> dict:
+    def load_from_disk(self, **kwargs):
         if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)
-
-    def get_gold_answer(self, input_d: dict) -> dict:
-        # return prompt and elements dict
-        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
-
-    def match(self, gold: dict, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
-
-            res[metric_name] = score.cpu().item()
-
-            # fine-granular metrics
-            if gold['tags'].get('advanced'):
-                res[f'{metric_name}_advanced'] = score.cpu().item()
-            else:
-                res[f'{metric_name}_basic'] = score.cpu().item()
-
-        return res
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record) -> Sample:
+        """Convert a record dictionary to a Sample object."""
+        advanced = record['tags'].get('advanced')
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'id': record['id'],
+                'prompt': record['prompt'],
+                'category': 'advanced' if advanced else 'basic',
+                'tags': record.get('tags', []),
+                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+            }
+        )

evalscope/benchmarks/aigc/t2i/general_t2i_adapter.py

@@ -1,58 +1,42 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
+import os
 
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageSystem, ChatMessageUser
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='general_t2i',
-    dataset_id='general_t2i',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['default'],
-    metric_list=['PickScore'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_t2i',
+        dataset_id='general_t2i',
+        description='General Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['default'],
+        metric_list=['PickScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
 )
-class GeneralT2IAdapter(T2IBaseAdapter):
+class GeneralT2IAdapter(Text2ImageAdapter):
 
     def __init__(self, **kwargs):
 
         super().__init__(**kwargs)
 
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record):
+        return Sample(input=[ChatMessageUser(content=record['prompt'])], metadata={'image_path': record['image_path']})

evalscope/benchmarks/aigc/t2i/hpdv2_adapter.py

@@ -1,57 +1,47 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
+import os
 
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='hpdv2',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['HPDv2'],
-    metric_list=['HPSv2.1Score'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='hpdv2',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='HPDv2 Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['HPDv2'],
+        metric_list=['HPSv2.1Score'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
 )
-class HPDv2Adapter(T2IBaseAdapter):
+class HPDv2Adapter(Text2ImageAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def load(self, **kwargs) -> dict:
+    def load_from_disk(self, **kwargs):
         if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)
-
-    def get_gold_answer(self, input_d: dict) -> dict:
-        # return prompt and elements dict
-        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
-
-    def match(self, gold: dict, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
-
-            res[metric_name] = score.cpu().item()
-
-            # fine-granular metrics
-            category = gold['tags'].get('category')
-            if category:
-                res[f'{metric_name}_{category}'] = score.cpu().item()
-
-        return res
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record):
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'category': record.get('tags', {}).get('category', ''),
+                'tags': record.get('tags', {})
+            }
+        )

evalscope/benchmarks/aigc/t2i/tifa_adapter.py

@@ -1,37 +1,26 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='tifa160',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['TIFA-160'],
-    metric_list=['PickScore'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='tifa160',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='TIFA-160 Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['TIFA-160'],
+        metric_list=['PickScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
 )
-class TIFA_Adapter(T2IBaseAdapter):
+class TIFA_Adapter(Text2ImageAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-
-    def load(self, **kwargs) -> dict:
-        if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)