evalscope 0.5.5rc1__py3-none-any.whl → 0.6.0rc0__py3-none-any.whl

evalscope/backend/__init__.py

@@ -1,3 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.backend.opencompass.backend_manager import OpenCompassBackendManager

evalscope/backend/opencompass/tasks/eval_datasets.py

@@ -50,6 +50,7 @@ with read_base():
     from opencompass.configs.datasets.nq.nq_gen_c788f6 import nq_datasets
     from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
     from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
+    from opencompass.configs.datasets.cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
     from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
     # Note: to be supported

evalscope/backend/rag_eval/__init__.py

@@ -0,0 +1,4 @@
+from evalscope.backend.rag_eval.utils.embedding import EmbeddingModel
+from evalscope.backend.rag_eval.utils.llm import LLM, LocalLLM, ChatOpenAI
+from evalscope.backend.rag_eval.utils.clip import VisionModel
+from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager

evalscope/backend/rag_eval/backend_manager.py

@@ -0,0 +1,80 @@
+import os
+from typing import Optional, Union
+from evalscope.utils import is_module_installed, get_valid_list
+from evalscope.backend.base import BackendManager
+from evalscope.utils.logger import get_logger
+
+
+logger = get_logger()
+
+
+class RAGEvalBackendManager(BackendManager):
+    def __init__(self, config: Union[str, dict], **kwargs):
+        """BackendManager for VLM Evaluation Kit
+
+        Args:
+            config (Union[str, dict]): the configuration yaml-file or the configuration dictionary
+        """
+        super().__init__(config, **kwargs)
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f"Check `{module_name}` Installed")
+        else:
+            logger.error(f"Please install `{module_name}` first")
+
+    @staticmethod
+    def run_mteb(model_args, eval_args):
+        from evalscope.backend.rag_eval.cmteb import ModelArguments, EvalArguments
+        from evalscope.backend.rag_eval.cmteb import one_stage_eval, two_stage_eval
+
+        if len(model_args) > 2:
+            raise ValueError("Not support multiple models yet")
+
+        # Convert arguments to dictionary
+        model_args_list = [ModelArguments(**args).to_dict() for args in model_args]
+        eval_args = EvalArguments(**eval_args).to_dict()
+
+        if len(model_args_list) == 1:
+            one_stage_eval(model_args_list[0], eval_args)
+        else:  # len(model_args_list) == 2
+            two_stage_eval(model_args_list[0], model_args_list[1], eval_args)
+
+    @staticmethod
+    def run_ragas(testset_args, eval_args):
+        from evalscope.backend.rag_eval.ragas import rag_eval
+        from evalscope.backend.rag_eval.ragas.tasks import generate_testset
+        from evalscope.backend.rag_eval.ragas import (
+            TestsetGenerationArguments,
+            EvaluationArguments,
+        )
+
+        if testset_args is not None:
+            generate_testset(TestsetGenerationArguments(**testset_args))
+        if eval_args is not None:
+            rag_eval(EvaluationArguments(**eval_args))
+
+    @staticmethod
+    def run_clip_benchmark(args):
+        from evalscope.backend.rag_eval.clip_benchmark import Arguments, evaluate
+
+        evaluate(Arguments(**args))
+
+    def run(self, *args, **kwargs):
+        tool = self.config_d.pop("tool")
+        if tool.lower() == "mteb":
+            self._check_env("mteb")
+            model_args = self.config_d["model"]
+            eval_args = self.config_d["eval"]
+            self.run_mteb(model_args, eval_args)
+        elif tool.lower() == "ragas":
+            self._check_env("ragas")
+            testset_args = self.config_d.get("testset_generation", None)
+            eval_args = self.config_d.get("eval", None)
+            self.run_ragas(testset_args, eval_args)
+        elif tool.lower() == "clip_benchmark":
+            self._check_env("webdataset")
+            self.run_clip_benchmark(self.config_d["eval"])
+        else:
+            raise ValueError(f"Unknown tool: {tool}")

evalscope/backend/rag_eval/clip_benchmark/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.backend.rag_eval.clip_benchmark.task_template import evaluate
+from evalscope.backend.rag_eval.clip_benchmark.arguments import Arguments

evalscope/backend/rag_eval/clip_benchmark/arguments.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass, field
+from typing import List, Dict
+
+
+@dataclass
+class Arguments:
+    # fmt: off
+    """
+    A dataclass to store and manage the arguments for the model configuration and data processing.
+    """
+    """
+    For CLIP model support, you can use the following fields:
+        model_name: str
+        revision: str = "master"
+        hub: str = "modelscope"
+        
+    For API VLM model support, you can use the following fields, (image caption only):
+        model_name="gpt-4o-mini"
+        api_base: str = "",
+        api_key: Optional[str] = None
+        prompt: str = None
+    """
+    models: List[Dict] = field(default_factory=dict)  # List of paths to the pre-trained models or model identifiers
+    dataset_name: List[str] = field(default_factory=list)  # List of dataset names to be used
+    data_dir: str = None  # Root directory where the datasets are stored
+    split: str = "test"  # Split of the dataset to be used (e.g., 'train', 'validation', 'test')
+    task: str = None
+    batch_size: int = 128  # Batch size for data loading
+    num_workers: int = 1  # Number of workers for data loading
+    verbose: bool = True  # Flag to enable verbose logging
+    output_dir: str = "outputs"  # Directory where the outputs (e.g., predictions, logs) will be saved
+    cache_dir: str = "cache"  # Directory where the dataset cache will be stored
+    skip_existing: bool = False  # Flag to skip processing if outputs already exist
+    limit: int = None # Limit the number of samples to be processed

evalscope/backend/rag_eval/clip_benchmark/dataset_builder.py

@@ -0,0 +1,277 @@
+import os
+import torch
+from torch.utils.data import DataLoader, Dataset as TorchDataset
+from evalscope.utils.logger import get_logger
+
+
+logger = get_logger()
+
+
+def build_dataset(
+    dataset_name,
+    root=None,
+    transform=None,
+    split="test",
+    wds_cache_dir=None,
+    **kwargs,
+):
+    """
+    Main function to use in order to build a dataset instance,
+
+    dataset_name: str
+        name of the dataset
+
+    root: str
+        root folder where the dataset is downloaded and stored. can be shared among datasets.
+
+    transform: torchvision transform applied to images
+
+    split: str
+        split to use, depending on the dataset can have different options.
+        In general, `train` and `test` are available.
+        For specific splits, please look at the corresponding dataset.
+
+    custom_classname_file: str or None
+        Custom classname file where keys are dataset names and values are list of classnames.
+
+    custom_template_file: str or None
+        Custom template file where keys are dataset names and values are list of prompts, or dicts
+        where keys are classnames and values are class-specific prompts.
+
+    """
+
+    if dataset_name == "dummy":
+        ds = Dummy()
+    elif dataset_name == "custom":
+        ds = build_custom_dataset(dataset_name, data_dir=root, transform=transform)
+    else:
+        # WebDataset support using `webdataset` library
+        ds = build_wds_dataset(
+            dataset_name,
+            transform=transform,
+            split=split,
+            data_dir=root,
+            cache_dir=wds_cache_dir,
+        )
+
+    return ds
+
+
+class Dummy:
+
+    def __init__(self):
+        self.classes = ["blank image", "noisy image"]
+
+    def __getitem__(self, i):
+        return torch.zeros(3, 224, 224), 0
+
+    def __len__(self):
+        return 1
+
+
+class DatasetWrapper(TorchDataset):
+    def __init__(self, dataset, transform=None, image_key="image", text_key="query"):
+        self.dataset = dataset
+        self.transform = transform
+        self.image_key = image_key
+        self.text_key = text_key
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, idx):
+        item = self.dataset[idx]
+
+        # 加载图像
+        image = item[self.image_key]
+        if self.transform is not None:
+            image = self.transform(image, return_tensors="pt")
+
+        # 获取查询列表
+        query = item[self.text_key]
+        if isinstance(query, str):
+            query = [query]
+
+        return image, query
+
+
+def get_dataset_default_task(dataset):
+    if dataset in (
+        "custom",
+        "muge",
+        "flickr30k",
+        "flickr8k",
+        "mscoco_captions",
+        "mscoco_captions2017",
+        "multilingual_mscoco_captions",
+        "flickr30k-200",
+        "crossmodal3600",
+        "xtd200",
+    ):
+        return "zeroshot_retrieval"
+    else:
+        return "zeroshot_classification"
+
+
+def get_dataloader(dataset_name, dataset, batch_size, num_workers):
+    if dataset_name == "custom":
+        dataloader = DataLoader(
+            dataset,
+            batch_size=batch_size,
+            shuffle=False,
+            num_workers=num_workers,
+            collate_fn=image_captions_collate_fn,
+        )
+    else:
+        dataloader = DataLoader(
+            dataset.batched(batch_size),
+            batch_size=None,
+            shuffle=False,
+            num_workers=num_workers,
+        )
+    return dataloader
+
+
+def image_captions_collate_fn(batch):
+    transposed = list(zip(*batch))
+    imgs = transposed[0]
+    texts = transposed[1]
+    return imgs, texts
+
+
+def build_custom_dataset(dataset_name, data_dir, transform=None):
+    from datasets import load_dataset, Features, Image, Sequence, Value
+
+    qrels_ds = load_dataset(
+        "json",
+        data_files=os.path.join(data_dir, "image_queries.jsonl"),
+        features=Features(
+            {"image_path": Image(decode=True), "query": Sequence(Value("string"))}
+        ),
+        split="train",
+    )
+
+    dataset = DatasetWrapper(
+        qrels_ds, transform, image_key="image_path", text_key="query"
+    )
+    return dataset
+
+
+def build_wds_dataset(
+    dataset_name, transform, split="test", data_dir="root", cache_dir=None
+):
+    """
+    Load a dataset in WebDataset format. Either local paths or HTTP URLs can be specified.
+    Expected file structure is:
+    ```
+    data_dir/
+        train/
+            nshards.txt
+            0.tar
+            1.tar
+            ...
+        test/
+            nshards.txt
+            0.tar
+            1.tar
+            ...
+        classnames.txt
+        zeroshot_classification_templates.txt
+        dataset_type.txt
+    ```
+    Classnames and templates are required for zeroshot classification, while dataset type
+    (equal to "retrieval") is required for zeroshot retrieval datasets.
+
+    You can use the `clip_benchmark_export_wds` or corresponding API
+    (`clip_benchmark.webdataset_builder.convert_dataset`) to convert datasets to this format.
+
+    Set `cache_dir` to a path to cache the dataset, otherwise, no caching will occur.
+    """
+    import webdataset as wds
+
+    def read_txt(fname):
+        if "://" in fname:
+            stream = os.popen("curl -L -s --fail '%s'" % fname, "r")
+            value = stream.read()
+            if stream.close():
+                raise FileNotFoundError("Failed to retreive data")
+        else:
+            with open(fname, "r") as file:
+                value = file.read()
+        return value
+
+    if not data_dir:
+        data_dir = f"https://modelscope.cn/datasets/clip-benchmark/wds_{dataset_name}/resolve/master"
+
+    # Git LFS files have a different file path to access the raw data than other files
+    if data_dir.startswith("https://modelscope.cn/datasets"):
+        *split_url_head, _, url_path = data_dir.split("/", 7)
+        url_head = "/".join(split_url_head)
+        metadata_dir = "/".join([url_head, "resolve", url_path])
+        tardata_dir = "/".join([url_head, "resolve", url_path])
+    else:
+        metadata_dir = tardata_dir = data_dir
+    # Get number of shards
+    nshards_fname = os.path.join(metadata_dir, split, "nshards.txt")
+    nshards = int(
+        read_txt(nshards_fname)
+    )  # Do not catch FileNotFound, nshards.txt should be mandatory
+    
+    # Get dataset type (classification or retrieval)
+    type_fname = os.path.join(metadata_dir, "dataset_type.txt")
+    try:
+        dataset_type = read_txt(type_fname).strip().lower()
+    except FileNotFoundError:
+        dataset_type = "classification"
+    
+    filepattern = os.path.join(tardata_dir, split, "{0..%d}.tar" % (nshards - 1))
+    # Load webdataset (support WEBP, PNG, and JPG for now)
+    if not cache_dir or not isinstance(cache_dir, str):
+        cache_dir = None
+    else:
+        os.makedirs(cache_dir, exist_ok=True)
+    dataset = wds.WebDataset(
+        filepattern,
+        cache_dir=cache_dir,
+        nodesplitter=lambda src: src,
+        shardshuffle=False,
+        verbose=True,
+    ).decode(
+        wds.autodecode.ImageHandler("pil", extensions=["webp", "png", "jpg", "jpeg"])
+    )
+    
+    # Load based on classification or retrieval task
+    if dataset_type == "retrieval":
+        dataset = dataset.to_tuple(["webp", "png", "jpg", "jpeg"], "txt").map_tuple(
+            transform, str.splitlines
+        )
+        dataset.classes = dataset.templates = None
+    else:
+        label_type = (
+            "npy" if dataset_type == "multilabel" else "cls"
+        )  # Special case for multilabel
+        dataset = dataset.to_tuple(
+            ["webp", "png", "jpg", "jpeg"], label_type
+        ).map_tuple(transform, None)
+        # Get class names if present
+        classnames_fname = os.path.join(metadata_dir, "classnames.txt")
+        try:
+            dataset.classes = [
+                line.strip() for line in read_txt(classnames_fname).splitlines()
+            ]
+        except FileNotFoundError:
+            logger.warning("WARNING: classnames.txt not found")
+            dataset.classes = None
+        # Get zeroshot classification templates if present
+        templates_fname = os.path.join(
+            metadata_dir, "zeroshot_classification_templates.txt"
+        )
+        try:
+            dataset.templates = [
+                line.strip() for line in read_txt(templates_fname).splitlines()
+            ]
+        except FileNotFoundError:
+            logger.warning("WARNING: zeroshot_classification_templates.txt not found")
+            dataset.templates = None
+
+    return dataset

evalscope/backend/rag_eval/clip_benchmark/task_template.py

@@ -0,0 +1,119 @@
+import os
+import torch
+import json
+from itertools import product
+
+from evalscope.backend.rag_eval.clip_benchmark.dataset_builder import (
+    build_dataset,
+    get_dataset_default_task,
+    get_dataloader,
+)
+from evalscope.backend.rag_eval.clip_benchmark.tasks import (
+    zeroshot_classification,
+    zeroshot_retrieval,
+    image_caption,
+)
+from evalscope.backend.rag_eval.clip_benchmark.arguments import Arguments
+from evalscope.backend.rag_eval.utils.clip import VisionModel
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def evaluate(args: Arguments):
+    models = args.models
+    dataset_names = args.dataset_name
+    data_dir = args.data_dir
+    split = args.split
+    batch_size = args.batch_size
+    num_workers = args.num_workers
+    verbose = args.verbose
+    input_task = args.task
+    output_dir = args.output_dir
+    cache_dir = args.cache_dir
+    skip_existing = args.skip_existing
+    limit = args.limit
+
+    # Iterate over model and dataset combinations
+    for model_cfg, dataset_name in product(models, dataset_names):
+        task = input_task or get_dataset_default_task(dataset_name)
+        model_name = os.path.basename(model_cfg["model_name"])
+
+        output_path = os.path.join(output_dir, model_name)
+        os.makedirs(output_path, exist_ok=True)
+        output_file = os.path.join(output_path, f"{dataset_name}_{task}.json")
+
+        # Skip evaluation if the result already exists and skip_existing is True
+        if os.path.exists(output_file) and skip_existing:
+            if verbose:
+                logger.info(f"Skip {output_dir}, exists already.")
+            return
+
+        # Determine device (CPU or GPU)
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        model_cfg["device"] = device
+        # Initialize the model
+        model = VisionModel.load(**model_cfg)
+
+        # Build the dataset
+        dataset = build_dataset(
+            dataset_name=dataset_name,
+            root=data_dir,
+            transform=model.transform,
+            split=split,
+            wds_cache_dir=f"{cache_dir}/{dataset_name}",
+        )
+
+        # Create the dataloader
+        dataloader = get_dataloader(dataset_name, dataset, batch_size, num_workers)
+
+        # Evaluate based on the task
+        if task == "zeroshot_classification":
+            zeroshot_templates = (
+                dataset.templates if hasattr(dataset, "templates") else None
+            )
+            if verbose:
+                logger.info(f"Zero-shot templates: {zeroshot_templates}")
+            classnames = dataset.classes if hasattr(dataset, "classes") else None
+            assert (
+                zeroshot_templates is not None and classnames is not None
+            ), "Dataset does not support classification"
+            metrics = zeroshot_classification.evaluate(
+                model,
+                dataloader,
+                classnames,
+                zeroshot_templates,
+                device=device,
+                verbose=verbose,
+                limit=limit,
+            )
+        elif task == "zeroshot_retrieval":
+            metrics = zeroshot_retrieval.evaluate(
+                model, dataloader, recall_k_list=[5], device=device, limit=limit
+            )
+        elif task == "image_caption":
+            output_path = os.path.join(output_path, dataset_name, "retrieval_data")
+            metrics = image_caption.evaluate(
+                model, dataloader, limit=limit, output_path=output_path
+            )
+
+        # Prepare dump data
+        dump = {
+            "dataset": dataset_name,
+            "model": model_name,
+            "task": task,
+            "metrics": metrics,
+        }
+
+        if verbose:
+            logger.info(f"Evaluation results: {dump}")
+
+        # Write the results to output file
+        if verbose:
+            logger.info(f"Dump results to: {output_file}")
+        with open(output_file, "w") as f:
+            json.dump(dump, f)
+
+
+if __name__ == "__main__":
+    evaluate()

evalscope/backend/rag_eval/clip_benchmark/tasks/image_caption.py

@@ -0,0 +1,83 @@
+from tqdm import tqdm
+import pandas as pd
+import os
+from evalscope.backend.rag_eval.utils.tools import save_to_jsonl, save_to_tsv
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def evaluate(model, dataloader, limit=None, output_path=""):
+    """
+    Evaluate the model on the dataset
+    Parameters
+    ----------
+    model: MultiModalModel
+        model to caption the image
+    dataloader: torch.utils.data.Dataloader
+    limit: int
+        limit the number of samples to evaluate
+    Returns
+    -------
+    dict of retrieval metrics
+    """
+    sample_count = 0
+    dataloader = dataloader_with_indices(dataloader)
+    query_caption_index = []
+    total_captions = []
+    total_querys = []
+    for batch_images, batch_texts, inds in tqdm(dataloader):
+        captions = model.encode_image(batch_images)
+        querys = [text for texts in batch_texts for text in texts]
+
+        batch_texts_image_index = [
+            ind for ind, texts in zip(inds, batch_texts) for text in texts
+        ]
+
+        total_captions.extend(captions)
+        total_querys.extend(querys)
+        query_caption_index.extend(batch_texts_image_index)
+
+        if limit is not None:
+            # Update sample counter
+            sample_count += len(batch_images)
+
+            if sample_count >= limit:
+                break
+
+    write_file(total_querys, total_captions, query_caption_index, output_path)
+    return {"convertion_successful": True, "save_path": output_path}
+
+
+def write_file(query_list, corpus_list, qrels_list, output_path):
+    # 处理 query_list
+    query_df = pd.DataFrame(query_list, columns=["text"])
+    query_df["_id"] = query_df.index
+    query_df = query_df[["_id", "text"]]
+    save_to_jsonl(query_df, os.path.join(output_path, "queries.jsonl"))
+
+    # 处理 corpus_list
+    corpus_df = pd.DataFrame(corpus_list, columns=["text"])
+    corpus_df["_id"] = corpus_df.index
+    corpus_df = corpus_df[["_id", "text"]]
+    save_to_jsonl(corpus_df, os.path.join(output_path, "corpus.jsonl"))
+
+    # 处理 qrels_list
+    qrels_df = pd.DataFrame(qrels_list, columns=["corpus-id"])
+    qrels_df["query-id"] = qrels_df.index
+    qrels_df["score"] = 1
+    qrels_df = qrels_df[["query-id", "corpus-id", "score"]]
+    save_to_tsv(qrels_df, os.path.join(output_path, "qrels", "test.tsv"))
+
+    logger.info("Write files to {}".format(output_path))
+    return
+
+
+def dataloader_with_indices(dataloader):
+    start = 0
+    for x, y in dataloader:
+        end = start + len(x)
+        inds = list(range(start, end))
+        yield x, y, inds
+        start = end