evalscope 0.5.5rc0__py3-none-any.whl → 0.6.0__py3-none-any.whl

evalscope/backend/rag_eval/clip_benchmark/tasks/zeroshot_classification.py

@@ -0,0 +1,247 @@
+"""
+Code adapated from https://github.com/mlfoundations/open_clip/blob/main/src/training/zero_shot.py
+Thanks to the authors of OpenCLIP
+"""
+
+import logging
+from contextlib import suppress
+
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+
+from sklearn.metrics import classification_report, balanced_accuracy_score
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def zero_shot_classifier(model, classnames, templates, device, amp=True):
+    """
+    This function returns zero-shot vectors for each class in order
+    to use it for zero-shot classification.
+
+
+    model:
+        CLIP-like model with `encode_text`
+
+    classnames: list of str
+        name of classes
+
+    templates: list of str
+        templates to use.
+
+    Returns
+    -------
+
+    torch.Tensor of shape (N,C) where N is the number
+    of templates, and C is the number of classes.
+    """
+    autocast = torch.amp.autocast if amp else suppress
+    with torch.no_grad(), autocast(device):
+        zeroshot_weights = []
+        for classname in tqdm(classnames):
+            if type(templates) == dict:
+                # class-specific prompts (e.g., CuPL https://arxiv.org/abs/2209.03320)
+                texts = templates[classname]
+            elif type(templates) == list:
+                # generic prompts tht are specialized for each class by replacing {c} with the class name
+                texts = [template.format(c=classname) for template in templates]
+            else:
+                raise ValueError("templates must be a list or a dict")
+            class_embedding = model.encode_text(texts).mean(dim=0)
+            class_embedding = F.normalize(class_embedding, dim=0)
+            zeroshot_weights.append(class_embedding)
+        zeroshot_weights = torch.stack(zeroshot_weights, dim=1).to(device)
+    return zeroshot_weights
+
+
+def accuracy(output, target, topk=(1,)):
+    """
+    Compute top-k accuracy
+
+    output: torch.Tensor
+        shape (N, C) where N is the number of examples, C the number of classes.
+        these are the logits.
+
+    target: torch.Tensor
+        shape (N,) where N is the number of examples. Groundtruth class id of each example.
+
+    topk: tuple
+        which topk to compute, e.g., topk=(1,5) will compute top-1 and top-5 accuracies
+
+    Returns
+    -------
+
+    list of top-k accuracies in the same order as `topk`
+    """
+    pred = output.topk(max(topk), 1, True, True)[1].t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+    n = len(target)
+    return [
+        float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) / n
+        for k in topk
+    ]
+
+
+def run_classification(model, classifier, dataloader, device, amp=True, limit=None):
+    """
+    Run zero-shot classifcation
+
+    model: torch.nn.Module
+        CLIP-like model with `encode_image` and `encode_text`
+
+    classifier: torch.Tensor
+        obtained from the function `zero_shot_classifier`
+
+    dataloader: torch.utils.data.Dataloader
+
+    Returns
+    -------
+    (pred, true)  where
+        - pred (N, C) are the logits
+        - true (N,) are the actual classes
+    """
+    autocast = torch.amp.autocast if amp else suppress
+    pred = []
+    true = []
+    sample_count = 0
+    with torch.no_grad():
+        for images, target in tqdm(dataloader):
+            target = target.to(device)
+
+            with autocast(device):
+                # predict
+                image_features = model.encode_image(images)
+                logits = 100.0 * image_features @ classifier
+            
+            if limit is not None:
+                # Update sample counter
+                sample_count += len(images)
+
+                if sample_count >= limit:
+                    break
+
+            true.append(target.cpu())
+            pred.append(logits.float().cpu())
+
+    pred = torch.cat(pred)
+    true = torch.cat(true)
+    return pred, true
+
+
+def average_precision_per_class(scores, targets):
+    """
+    Compute average precision  for each class
+    this metric is used for multi-label classification
+    see explanations here https://fangdahan.medium.com/calculate-mean-average-precision-map-for-multi-label-classification-b082679d31be
+    Code is adapted from https://github.com/pytorch/tnt/blob/master/torchnet/meter/meter.py, thanks to the authors of `tnt`.
+
+    Parameters
+    ----------
+
+    scores: torch.Tensor
+        logits, of shape (N,C) where N is the number of examples, C the number of classes
+
+    targets: torch.Tensor
+        one-hot vectors of groundtruth targets (N, C), where N is the number of examples, C is the
+        number of classes
+
+    Returns
+    -------
+
+    torch.Tensor of shape (C,) of avereage precision for each class, where C is
+    the number of classes.
+
+    """
+    ap = torch.zeros(scores.size(1))
+    rg = torch.arange(1, scores.size(0) + 1).float()
+    # compute average precision for each class
+    for k in range(scores.size(1)):
+        # sort scores
+        scores_k = scores[:, k]
+        targets_k = targets[:, k]
+        _, sortind = torch.sort(scores_k, 0, True)
+        truth = targets_k[sortind]
+        tp = truth.float().cumsum(0)
+        # compute precision curve
+        precision = tp.div(rg)
+        # compute average precision
+        ap[k] = precision[truth.bool()].sum() / max(float(truth.sum()), 1)
+    return ap
+
+
+def evaluate(
+    model,
+    dataloader,
+    classnames,
+    templates,
+    device,
+    amp=True,
+    verbose=False,
+    limit=None,
+):
+    """
+    Run zero-shot classification and evaluate the metrics
+
+    Parameters
+    ----------
+
+    model: torch.nn.Module
+        CLIP-like model with `encode_image` and `encode_text`
+
+    dataloader: torch.utils.data.Dataloader
+
+    classnames: list of str
+        class names
+
+    templates: list of str
+        templates to use for zero-shot classification
+
+    device: cpu/cuda
+
+    amp: whether to use automatic mixed precision
+
+    verbose: whether to use verbose model
+
+    Returns
+    -------
+
+    dict of classification metrics
+    """
+    classifier = zero_shot_classifier(model, classnames, templates, device, amp=amp)
+
+    logits, target = run_classification(model, classifier, dataloader, device, amp=amp, limit=limit)
+    is_multilabel = len(target.shape) == 2
+
+    if is_multilabel:
+        if verbose:
+            logger.info("Detected a multi-label classification dataset")
+        # Multiple labels per image, multiple classes on the dataset
+        ap_per_class = average_precision_per_class(logits, target)
+        if verbose:
+            for class_name, ap in zip(
+                dataloader.dataset.classes, ap_per_class.tolist()
+            ):
+                logger.info(f"Class: {class_name}, AveragePrecision: {ap}")
+        return {"mean_average_precision": ap_per_class.mean().item()}
+    else:
+        # Single label per image, multiple classes on the dataset
+        # just compute accuracy and mean_per_class_recall
+
+        pred = logits.argmax(axis=1)
+        # measure accuracy
+        if len(dataloader.dataset.classes) >= 5:
+            acc1, acc5 = accuracy(logits, target, topk=(1, 5))
+        else:
+            (acc1,) = accuracy(logits, target, topk=(1,))
+            acc5 = float("nan")
+        mean_per_class_recall = balanced_accuracy_score(target, pred)
+        if verbose:
+            logger.info("\n" + classification_report(target, pred, digits=3))
+        return {
+            "acc1": acc1,
+            "acc5": acc5,
+            "mean_per_class_recall": mean_per_class_recall,
+        }

evalscope/backend/rag_eval/clip_benchmark/tasks/zeroshot_retrieval.py

@@ -0,0 +1,170 @@
+import logging
+from contextlib import suppress
+
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def evaluate(model, dataloader, device, amp=True, recall_k_list=[5], limit=None):
+    """
+    Evaluate the model on the given dataset
+
+    Parameters
+    ----------
+
+    model: torch.nn.Module
+        CLIP-like model with `encode_image` and `encode_text`
+
+    dataloader: torch.utils.data.Dataloader
+        dataloader to use for evaluation
+
+    tokenizer:
+        text tokenizer, i.e. convert list of strings to torch.Tensor of integers
+
+    device: cpu/cuda
+
+    amp: whether to use automatic mixed precision
+
+    recall_k_list: list of int
+        recall@k k's to use
+
+    limit: int
+        maximum number of samples to evaluate
+
+    Returns
+    -------
+
+    dict of retrieval metrics
+    """
+    # list of batch of images embedding
+    batch_images_emb_list = []
+    # list of batch of text embedding
+    batch_texts_emb_list = []
+    # for each text, we collect the corresponding image index, as each image can have multiple corresponding texts
+    texts_image_index = []
+    sample_count = 0
+    dataloader = dataloader_with_indices(dataloader)
+    for batch_images, batch_texts, inds in tqdm(dataloader):
+
+        # store the index of image for each text
+        batch_texts_image_index = [
+            ind for ind, texts in zip(inds, batch_texts) for text in texts
+        ]
+
+        # compute the embedding of images and texts
+        batch_images_emb = model.encode_image(batch_images)
+        batch_texts_emb = model.encode_text(batch_texts)
+
+        batch_images_emb_list.append(batch_images_emb.cpu())
+        batch_texts_emb_list.append(batch_texts_emb.cpu())
+        texts_image_index.extend(batch_texts_image_index)
+
+        if limit is not None:
+            # Update sample counter
+            sample_count += len(batch_images)
+
+            if sample_count >= limit:
+                break
+
+    batch_size = len(batch_images_emb_list[0])
+
+    # concatenate all embeddings
+    images_emb = torch.cat(batch_images_emb_list)
+    texts_emb = torch.cat(batch_texts_emb_list)
+
+    # get the score for each text and image pair
+    scores = texts_emb @ images_emb.t()
+
+    # construct a the positive pair matrix, which tells whether each text-image pair is a positive or not
+    positive_pairs = torch.zeros_like(scores, dtype=bool)
+    positive_pairs[torch.arange(len(scores)), texts_image_index] = True
+    metrics = {}
+    for recall_k in recall_k_list:
+        # Note that recall_at_k computes **actual** recall i.e. nb_true_positive/nb_positives, where the number
+        # of true positives, e.g. for text retrieval, is, for each image,  the number of retrieved texts matching that image among the top-k.
+        # Also, the number of positives are the total number of texts matching the image in the dataset, as we have a set of captions
+        # for each image, that number will be greater than 1 for text retrieval.
+        # However, image/text retrieval recall@k, the way it is done in CLIP-like papers, is a bit different.
+        # recall@k, in CLIP-like papers, is, for each image, either 1 or 0. It is 1 if atleast one text matches the image among the top-k.
+        # so we can easily compute that using the actual recall, by checking whether there is at least one true positive,
+        # which would be the case if the recall is greater than 0. One we compute the recal for each image (or text), we average
+        # it over the dataset.
+        metrics[f"image_retrieval_recall@{recall_k}"] = (
+            (
+                batchify(
+                    recall_at_k, scores, positive_pairs, batch_size, device, k=recall_k
+                )
+                > 0
+            )
+            .float()
+            .mean()
+            .item()
+        )
+        metrics[f"text_retrieval_recall@{recall_k}"] = (
+            (
+                batchify(
+                    recall_at_k,
+                    scores.T,
+                    positive_pairs.T,
+                    batch_size,
+                    device,
+                    k=recall_k,
+                )
+                > 0
+            )
+            .float()
+            .mean()
+            .item()
+        )
+
+    return metrics
+
+
+def dataloader_with_indices(dataloader):
+    start = 0
+    for x, y in dataloader:
+        end = start + len(x)
+        inds = torch.arange(start, end)
+        yield x, y, inds
+        start = end
+
+
+def recall_at_k(scores, positive_pairs, k):
+    """
+    Compute the recall at k for each sample
+    :param scores: compability score between  text and image embeddings (nb texts, nb images)
+    :param k: number of images to consider per text, for retrieval
+    :param positive_pairs: boolean matrix of positive pairs (nb texts, nb images)
+    :return: recall at k averaged over all texts
+    """
+    nb_texts, nb_images = scores.shape
+    # for each text, sort according to image scores in decreasing order
+    topk_indices = torch.topk(scores, k, dim=1)[1]
+    # compute number of positives for each text
+    nb_positive = positive_pairs.sum(dim=1)
+    # nb_texts, k, nb_images
+    topk_indices_onehot = torch.nn.functional.one_hot(
+        topk_indices, num_classes=nb_images
+    )
+    # compute number of true positives
+    positive_pairs_reshaped = positive_pairs.view(nb_texts, 1, nb_images)
+    # a true positive means a positive among the topk
+    nb_true_positive = (topk_indices_onehot * positive_pairs_reshaped).sum(dim=(1, 2))
+    # compute recall at k
+    recall_at_k = nb_true_positive / nb_positive
+    return recall_at_k
+
+
+def batchify(func, X, Y, batch_size, device, *args, **kwargs):
+    results = []
+    for start in range(0, len(X), batch_size):
+        end = start + batch_size
+        x = X[start:end].to(device)
+        y = Y[start:end].to(device)
+        result = func(x, y, *args, **kwargs).cpu()
+        results.append(result)
+    return torch.cat(results)

evalscope/backend/rag_eval/cmteb/__init__.py

@@ -0,0 +1,4 @@
+from evalscope.backend.rag_eval.cmteb.tasks import *
+from evalscope.backend.rag_eval.cmteb.base import *
+from evalscope.backend.rag_eval.cmteb.arguments import ModelArguments, EvalArguments
+from evalscope.backend.rag_eval.cmteb.task_template import one_stage_eval, two_stage_eval

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Union, Dict, Any
+
+
+@dataclass
+class ModelArguments:
+    # Arguments for embeding model: sentence transformer or cross encoder
+    model_name_or_path: str = ""  # model name or path
+    is_cross_encoder: bool = False  # whether the model is a cross encoder
+    # pooling mode: Either “cls”, “lasttoken”, “max”, “mean”, “mean_sqrt_len_tokens”, or “weightedmean”.
+    pooling_mode: Optional[str] = None
+    max_seq_length: int = 512  # max sequence length
+    # prompt for llm based model
+    prompt: str = ""
+    # model kwargs
+    model_kwargs: dict = field(default_factory=dict)
+    # config kwargs
+    config_kwargs: Dict[str, Any] = field(default_factory=dict)
+    # encode kwargs
+    encode_kwargs: dict = field(
+        default_factory=lambda: {"show_progress_bar": True, "batch_size": 32}
+    )
+    hub: str = "modelscope"  # modelscope or huggingface
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "model_name_or_path": self.model_name_or_path,
+            "is_cross_encoder": self.is_cross_encoder,
+            "pooling_mode": self.pooling_mode,
+            "max_seq_length": self.max_seq_length,
+            "prompt": self.prompt,
+            "model_kwargs": self.model_kwargs,
+            "config_kwargs": self.config_kwargs,
+            "encode_kwargs": self.encode_kwargs,
+            "hub": self.hub,
+        }
+
+
+@dataclass
+class EvalArguments:
+    # Evaluation
+    tasks: List[str] = field(default_factory=list)  # task names
+    dataset_path: Optional[str] = None  # custom dataset path
+    verbosity: int = 2  # verbosity level 0-3
+    output_folder: str = "outputs"  # output folder
+    overwrite_results: bool = True  # overwrite results
+    limits: Optional[int] = None  # limit number of samples
+    hub: str = "modelscope"  # modelscope or huggingface
+    top_k: int = 5 # top k for reranking
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "tasks": self.tasks,
+            "dataset_path": self.dataset_path,
+            "verbosity": self.verbosity,
+            "output_folder": self.output_folder,
+            "overwrite_results": self.overwrite_results,
+            "limits": self.limits,
+            "hub": self.hub,
+            "top_k": self.top_k,
+        }

evalscope/backend/rag_eval/cmteb/base.py

@@ -0,0 +1,91 @@
+from collections import defaultdict
+from typing import List
+from mteb import AbsTask
+from datasets import DatasetDict
+from modelscope import MsDataset
+import datasets
+from evalscope.backend.rag_eval.cmteb.tasks import CLS_DICT, CLS_RETRIEVAL, CLS_CUSTOM
+
+__all__ = ["TaskBase"]
+
+
+class TaskBase:
+
+    @staticmethod
+    def get_tasks(task_names, **kwargs) -> List[AbsTask]:
+
+        return [TaskBase.get_task(task_name, **kwargs) for task_name in task_names]
+
+    @staticmethod
+    def get_task(task_name, **kwargs) -> AbsTask:
+
+        if task_name in CLS_CUSTOM:
+            task_cls = CLS_CUSTOM[task_name]
+        elif task_name in CLS_DICT:
+            task_cls = CLS_DICT[task_name]
+            task_cls.load_data = load_data
+        else:
+            from mteb.overview import TASKS_REGISTRY
+
+            task_cls = TASKS_REGISTRY[task_name]
+            if task_cls.metadata.type != "Retrieval":
+                task_cls.load_data = load_data
+
+        # init task instance
+        task_instance = task_cls(**kwargs)
+        return task_instance
+
+
+def load_data(self, **kwargs):
+    """Load dataset from the hub, compatible with ModelScope and Hugging Face."""
+    if self.data_loaded:
+        return
+
+    limits = kwargs.get("limits", None)
+    hub = kwargs.get("hub", "modelscope")
+    name = self.metadata_dict.get("name")
+    path = self.metadata_dict["dataset"].get("path")
+
+    assert path is not None, "Path must be specified in dataset"
+
+    # Loading the dataset based on the source hub
+    if hub == "modelscope":
+        import re
+
+        path = re.sub(r"^mteb/", "MTEB/", path)
+        dataset = MsDataset.load(path)
+    else:
+        dataset = datasets.load_dataset(**self.metadata_dict["dataset"])  # type: ignore
+
+    if limits is not None:
+        dataset = {
+            split: dataset[split].select(range(min(limits, len(dataset[split]))))
+            for split in dataset.keys()
+        }
+
+    if name in CLS_RETRIEVAL:
+        self.corpus, self.queries, self.relevant_docs = load_retrieval_data(
+            dataset,
+            path,
+            self.metadata_dict["eval_splits"],
+        )
+
+    self.dataset = dataset
+    self.dataset_transform()
+    self.data_loaded = True
+
+
+def load_retrieval_data(dataset, dataset_name: str, eval_splits: list) -> tuple:
+    eval_split = eval_splits[0]
+    qrels = MsDataset.load(dataset_name + "-qrels")[eval_split]
+
+    corpus = {e["id"]: {"text": e["text"]} for e in dataset["corpus"]}
+    queries = {e["id"]: e["text"] for e in dataset["queries"]}
+    relevant_docs = defaultdict(dict)
+    for e in qrels:
+        relevant_docs[e["qid"]][e["pid"]] = e["score"]
+
+    corpus = DatasetDict({eval_split: corpus})
+    queries = DatasetDict({eval_split: queries})
+    relevant_docs = DatasetDict({eval_split: relevant_docs})
+    return corpus, queries, relevant_docs

evalscope/backend/rag_eval/cmteb/task_template.py

@@ -0,0 +1,85 @@
+import os
+import mteb
+from evalscope.backend.rag_eval import EmbeddingModel
+from evalscope.backend.rag_eval import cmteb
+from mteb.task_selection import results_to_dataframe
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def show_results(output_folder, model, results):
+    model_name = model.mteb_model_meta.model_name_as_path()
+    revision = model.mteb_model_meta.revision
+
+    results_df = results_to_dataframe({model_name: {revision: results}})
+
+    save_path = os.path.join(
+        output_folder,
+        model_name,
+        revision,
+    )
+    logger.info(f"Evaluation results:\n{results_df.to_markdown()}")
+    logger.info(f"Evaluation results saved in {os.path.abspath(save_path)}")
+
+
+def one_stage_eval(
+    model_args,
+    eval_args,
+) -> None:
+    # load model
+    model = EmbeddingModel.load(**model_args)
+    custom_dataset_path = eval_args.pop("dataset_path", None)
+    # load task first to update instructions
+    tasks = cmteb.TaskBase.get_tasks(
+        task_names=eval_args["tasks"], dataset_path=custom_dataset_path
+    )
+    evaluation = mteb.MTEB(tasks=tasks)
+
+    # run evaluation
+    results = evaluation.run(model, **eval_args)
+
+    # save and log results
+    show_results(eval_args["output_folder"], model, results)
+
+
+def two_stage_eval(
+    model1_args,
+    model2_args,
+    eval_args,
+) -> None:
+    """a two-stage run with the second stage reading results saved from the first stage."""
+    # load model
+    dual_encoder = EmbeddingModel.load(**model1_args)
+    cross_encoder = EmbeddingModel.load(**model2_args)
+
+    first_stage_path = f"{eval_args['output_folder']}/stage1"
+    second_stage_path = f"{eval_args['output_folder']}/stage2"
+
+    tasks = cmteb.TaskBase.get_tasks(task_names=eval_args["tasks"])
+    for task in tasks:
+        evaluation = mteb.MTEB(tasks=[task])
+
+        # stage 1: run dual encoder
+        evaluation.run(
+            dual_encoder,
+            save_predictions=True,
+            output_folder=first_stage_path,
+            overwrite_results=True,
+            hub=eval_args["hub"],
+            limits=eval_args["limits"],
+        )
+        # stage 2: run cross encoder
+        results = evaluation.run(
+            cross_encoder,
+            top_k=eval_args["top_k"],
+            save_predictions=True,
+            output_folder=second_stage_path,
+            previous_results=f"{first_stage_path}/{task.metadata.name}_default_predictions.json",
+            overwrite_results=True,
+            hub=eval_args["hub"],
+            limits=eval_args["limits"],
+        )
+
+        # save and log results
+        show_results(second_stage_path, cross_encoder, results)