mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/__init__.py

@@ -3,14 +3,17 @@ from importlib.metadata import version
 from mteb import types
 from mteb.abstasks import AbsTask
 from mteb.abstasks.task_metadata import TaskMetadata
+from mteb.cache import ResultCache
 from mteb.deprecated_evaluator import MTEB
 from mteb.evaluate import evaluate
 from mteb.filter_tasks import filter_tasks
 from mteb.get_tasks import get_task, get_tasks
 from mteb.load_results import load_results
 from mteb.models import (
+    CacheBackendProtocol,
     CrossEncoderProtocol,
     EncoderProtocol,
+    IndexEncoderSearchProtocol,
     SearchProtocol,
     SentenceTransformerEncoderWrapper,
 )
@@ -27,8 +30,11 @@ __all__ = [
     "AbsTask",
     "Benchmark",
     "BenchmarkResults",
+    "CacheBackendProtocol",
     "CrossEncoderProtocol",
     "EncoderProtocol",
+    "IndexEncoderSearchProtocol",
+    "ResultCache",
     "SearchProtocol",
     "SentenceTransformerEncoderWrapper",
     "TaskMetadata",

mteb/_create_dataloaders.py

@@ -1,9 +1,10 @@
 import logging
+import warnings
 from collections.abc import Callable
 from typing import Any, cast
 
 import torch
-from datasets import Dataset
+from datasets import Dataset, Image
 from torch.utils.data import DataLoader, default_collate
 
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -22,12 +23,14 @@ logger = logging.getLogger(__name__)
 def _create_dataloader_from_texts(
     text: list[str],
     batch_size: int = 32,
+    **kwargs: Any,
 ) -> DataLoader[TextInput]:
     """Create a dataloader from a list of text.
 
     Args:
         text: A list of text to create a dataloader from.
         batch_size: Batch size for the dataloader.
+        kwargs: Not used, present catching extra arguments.
 
     Returns:
         A dataloader with the text.
@@ -111,11 +114,8 @@ def _create_text_dataloader_for_queries(
     )
 
 
-_warned_about_user_role = False
-
-
 def _convert_conv_history_to_query(
-    row: dict[str, list[str] | Conversation],
+    row: dict[str, str | list[str] | Conversation],
 ) -> dict[str, str | Conversation]:
     """Convert a conversation history to a single query string.
 
@@ -125,21 +125,18 @@ def _convert_conv_history_to_query(
     Returns:
         The updated row with the "query" and "text" fields set to the conversation string, and the "conversation" field set to the list of ConversationTurn.
     """
-    global _warned_about_user_role
-
     conversation = row["text"]
     # if it's a list of strings, just join them
     if isinstance(conversation, list) and isinstance(conversation[0], str):
-        conversation = cast(list[str], conversation)
-        conv_str = "; ".join(conversation)
+        conversation_ = cast(list[str], conversation)
+        conv_str = "; ".join(conversation_)
         current_conversation = [
-            ConversationTurn(role="user", content=message) for message in conversation
+            ConversationTurn(role="user", content=message) for message in conversation_
         ]
-        if not _warned_about_user_role:
-            logger.warning(
-                "Conversations are a list of strings. Used 'user' role for all turns."
-            )
-            _warned_about_user_role = True
+        warnings.warn(
+            "Conversations are a list of strings. Used 'user' role for all turns.",
+            category=UserWarning,
+        )
     # otherwise, it's a list of dictionaries, which we need to convert to strings
     elif isinstance(conversation, list) and isinstance(conversation[0], dict):
         conv = []
@@ -176,7 +173,7 @@ def _convert_conv_history_to_query(
 
     row["text"] = conv_str
     row["conversation"] = current_conversation
-    return row
+    return cast(dict[str, str | list[ConversationTurn]], row)
 
 
 def _create_dataloader_for_queries_conversation(
@@ -194,7 +191,8 @@ def _create_dataloader_for_queries_conversation(
     """
     return DataLoader(
         queries.map(
-            _convert_conv_history_to_query, desc="Converting conversations to queries"
+            _convert_conv_history_to_query,
+            desc="Converting conversations to queries",
         ),
         collate_fn=_custom_collate_fn,
         batch_size=batch_size,
@@ -244,14 +242,15 @@ def _prepare_image_dataset(
     transform: Callable[[Any], Any] | None = None,
 ) -> Dataset:
     """Prepare the image dataset by converting images to RGB and applying transformations."""
-    # If the dataset uses a different column name for images, rename it to "image".
     if (
         image_column_name
         and image_column_name in dataset.column_names
         and "image" not in dataset.column_names
     ):
         dataset = dataset.rename_column(image_column_name, "image")
-    # Map the conversion function over the dataset.
+    # don't process image if it's already in the correct format
+    if isinstance(dataset.features["image"], Image):
+        return dataset
     return dataset.map(
         _convert_images_to_rgb,
         fn_kwargs={"image_col_name": "image", "transform": transform},
@@ -363,6 +362,9 @@ def _create_document_dataloader(
         task_metadata: Metadata of the task to determine the document type.
         input_column: The column to use as input. If None, it will use the first column that matches the modality.
         batch_size: Batch size for the dataloader.
+
+    Returns:
+        A dataloader for the documents.
     """
     document_type = task_metadata.get_modalities(PromptType.document)
     if document_type == ["text"]:  # text only
@@ -385,7 +387,7 @@ def create_dataloader(
     prompt_type: PromptType | None = None,
     input_column: str | None = None,
     batch_size: int = 32,
-    **kwargs: dict[str, Any],
+    **kwargs: Any,
 ) -> DataLoader[BatchedInput]:
     """Create a dataloader from a dataset.
 

mteb/_evaluators/any_sts_evaluator.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, TypedDict
+from typing import TypedDict
 
 from datasets import Dataset
 from sklearn.metrics.pairwise import (
@@ -12,6 +12,7 @@ from mteb._create_dataloaders import create_dataloader
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import EncoderProtocol
 from mteb.similarity_functions import compute_pairwise_similarity
+from mteb.types import EncodeKwargs, PromptType
 
 from .evaluator import Evaluator
 
@@ -42,44 +43,52 @@ class AnySTSEvaluator(Evaluator):
         task_metadata: TaskMetadata,
         hf_split: str,
         hf_subset: str,
+        input1_prompt_type: PromptType | None,
+        input2_prompt_type: PromptType | None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
-        self.first_column = create_dataloader(
-            dataset,
-            task_metadata,
-            input_column=sentences_column_names[0],
-        )
-        self.second_column = create_dataloader(
-            dataset,
-            task_metadata,
-            input_column=sentences_column_names[1],
-        )
+        self.dataset = dataset
+        self.input_columns = sentences_column_names
         self.task_metadata = task_metadata
         self.hf_split = hf_split
         self.hf_subset = hf_subset
+        self.input1_prompt_type = input1_prompt_type
+        self.input2_prompt_type = input2_prompt_type
 
     def __call__(
         self,
         model: EncoderProtocol,
         *,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
     ) -> STSEvaluatorScores:
         logger.info("Running semantic similarity - Encoding samples (1/2)")
         embeddings1 = model.encode(
-            self.first_column,
+            create_dataloader(
+                self.dataset,
+                self.task_metadata,
+                input_column=self.input_columns[0],
+                **encode_kwargs,
+            ),
             task_metadata=self.task_metadata,
             hf_split=self.hf_split,
             hf_subset=self.hf_subset,
+            prompt_type=self.input1_prompt_type,
             **encode_kwargs,
         )
 
         logger.info("Running semantic similarity - Encoding samples (2/2)...")
         embeddings2 = model.encode(
-            self.second_column,
+            create_dataloader(
+                self.dataset,
+                self.task_metadata,
+                input_column=self.input_columns[1],
+                **encode_kwargs,
+            ),
             task_metadata=self.task_metadata,
             hf_split=self.hf_split,
             hf_subset=self.hf_subset,
+            prompt_type=self.input2_prompt_type,
             **encode_kwargs,
         )
 

mteb/_evaluators/classification_metrics.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+
+def hamming_score(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+    """Compute the Hamming score (a.k.a. label-based accuracy) for multilabel classification.
+
+    The Hamming score is the fraction of labels that are correctly predicted for each sample,
+    averaged over all samples. For samples where both y_true and y_pred have no labels,
+    the score is 1.0 (perfect agreement).
+
+    Args:
+        y_true: Binary matrix of true labels with shape (n_samples, n_labels)
+        y_pred: Binary matrix of predicted labels with shape (n_samples, n_labels)
+
+    Returns:
+        float: Hamming score between 0.0 and 1.0
+
+    Raises:
+        ValueError: If inputs are invalid or have incompatible shapes
+        TypeError: If inputs cannot be converted to numpy arrays
+    """
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # Check shapes
+    if y_true.shape != y_pred.shape:
+        raise ValueError(
+            f"Shape mismatch: y_true {y_true.shape} != y_pred {y_pred.shape}"
+        )
+
+    # Check if arrays are empty
+    if y_true.size == 0:
+        raise ValueError("Input arrays cannot be empty")
+
+    # Ensure 2D arrays
+    if y_true.ndim != 2:
+        raise ValueError(f"Arrays must be 2D, got {y_true.ndim}D")
+
+    # Check for binary values
+    if not (np.all(np.isin(y_true, [0, 1])) and np.all(np.isin(y_pred, [0, 1]))):
+        raise ValueError("Arrays must contain only binary values (0 and 1)")
+
+    # Convert to boolean for bitwise operations
+    y_true_bool = y_true.astype(bool)
+    y_pred_bool = y_pred.astype(bool)
+
+    # Calculate intersection and union for each sample
+    intersection = (y_true_bool & y_pred_bool).sum(axis=1)
+    union = (y_true_bool | y_pred_bool).sum(axis=1)
+
+    # Handle division by zero: when union is 0, both are all zeros, so score is 1.0
+    scores = np.where(union == 0, 1.0, intersection / union)
+
+    return float(scores.mean())

mteb/_evaluators/clustering_evaluator.py

@@ -1,5 +1,4 @@
 import logging
-from typing import Any
 
 from datasets import Dataset
 from sklearn import cluster
@@ -7,6 +6,7 @@ from sklearn import cluster
 from mteb._create_dataloaders import create_dataloader
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import EncoderProtocol
+from mteb.types import EncodeKwargs
 
 from .evaluator import Evaluator
 
@@ -38,13 +38,13 @@ class ClusteringEvaluator(Evaluator):
         self,
         model: EncoderProtocol,
         *,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
     ) -> list[int]:
         data_loader = create_dataloader(
             self.dataset,
             self.task_metadata,
             input_column=self.input_column_name,
-            batch_size=encode_kwargs["batch_size"],
+            **encode_kwargs,
         )
 
         logger.info("Running clustering - Encoding samples...")

mteb/_evaluators/evaluator.py

@@ -1,8 +1,10 @@
 from abc import ABC, abstractmethod
+from collections.abc import Iterable, Mapping
 from typing import Any
 
 from mteb.abstasks.abstask import _set_seed
 from mteb.models import EncoderProtocol
+from mteb.types import EncodeKwargs
 
 
 class Evaluator(ABC):
@@ -17,8 +19,8 @@ class Evaluator(ABC):
 
     @abstractmethod
     def __call__(
-        self, model: EncoderProtocol, *, encode_kwargs: dict[str, Any]
-    ) -> dict[str, float]:
+        self, model: EncoderProtocol, *, encode_kwargs: EncodeKwargs
+    ) -> Mapping[str, float] | Iterable[Any]:
         """This is called during training to evaluate the model.
 
         It returns scores.

mteb/_evaluators/image/imagetext_pairclassification_evaluator.py

@@ -1,19 +1,26 @@
+from __future__ import annotations
+
 import logging
-from typing import Any
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any
 
 import torch
 import torch.nn.functional as F
-from datasets import Dataset
-from PIL.Image import Image
 from torch.utils.data import DataLoader
 
 from mteb._create_dataloaders import (
+    _create_dataloader_from_texts,
     _transform_image_to_rgb,
 )
 from mteb._evaluators.evaluator import Evaluator
 from mteb._requires_package import requires_image_dependencies
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.models_protocols import EncoderProtocol
+from mteb.types import EncodeKwargs
+
+if TYPE_CHECKING:
+    from PIL.Image import Image
+
 
 logger = logging.getLogger(__name__)
 
@@ -56,8 +63,8 @@ class ImageTextPairClassificationEvaluator(Evaluator):
     def __init__(
         self,
         dataset,
-        images_column_names: str | list[str],
-        texts_column_names: str | list[str],
+        images_column_names: str | Sequence[str],
+        texts_column_names: str | Sequence[str],
         num_images_per_sample: int,
         num_texts_per_sample: int,
         task_metadata: TaskMetadata,
@@ -77,10 +84,11 @@ class ImageTextPairClassificationEvaluator(Evaluator):
         self.hf_split = hf_split
         self.hf_subset = hf_subset
 
-    def __call__(
+    def __call__(  # type: ignore[override]
         self,
         model: EncoderProtocol,
-        encode_kwargs: dict[str, Any],
+        *,
+        encode_kwargs: EncodeKwargs,
     ) -> list[torch.Tensor]:
         images = []
         if isinstance(self.images_column_names, str):
@@ -101,9 +109,9 @@ class ImageTextPairClassificationEvaluator(Evaluator):
                     texts.append(row[col])
 
         text_embeddings = model.encode(
-            DataLoader(
-                Dataset.from_dict({"text": texts}),
-                batch_size=encode_kwargs["batch_size"],
+            _create_dataloader_from_texts(
+                texts,
+                **encode_kwargs,
             ),
             task_metadata=self.task_metadata,
             hf_subset=self.hf_subset,
@@ -122,7 +130,6 @@ class ImageTextPairClassificationEvaluator(Evaluator):
         image_embeddings = model.encode(
             DataLoader(
                 CustomImageDataset(images),
-                batch_size=encode_kwargs["batch_size"],
                 collate_fn=lambda x: {"image": [item["image"] for item in x]},
             ),
             task_metadata=self.task_metadata,

mteb/_evaluators/pair_classification_evaluator.py

@@ -14,6 +14,7 @@ from mteb._evaluators.evaluator import Evaluator
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import EncoderProtocol
 from mteb.similarity_functions import compute_pairwise_similarity
+from mteb.types import EncodeKwargs, PromptType
 
 logger = logging.getLogger(__name__)
 
@@ -60,6 +61,8 @@ class PairClassificationEvaluator(Evaluator):
         task_metadata: TaskMetadata,
         hf_split: str,
         hf_subset: str,
+        input1_prompt_type: PromptType | None,
+        input2_prompt_type: PromptType | None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -69,6 +72,8 @@ class PairClassificationEvaluator(Evaluator):
         self.task_metadata = task_metadata
         self.hf_split = hf_split
         self.hf_subset = hf_subset
+        self.input1_prompt_type = input1_prompt_type
+        self.input2_prompt_type = input2_prompt_type
 
         if len(self.dataset[self.input1_column_name]) != len(
             self.dataset[self.input2_column_name]
@@ -80,49 +85,36 @@ class PairClassificationEvaluator(Evaluator):
     def __call__(
         self,
         model: EncoderProtocol,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
     ) -> PairClassificationDistances:
-        logger.info("Running pair classification - Encoding inputs...")
-        if self.task_metadata.modalities == ["text"]:
-            # datasets v4 will pass column objects, so we need to extract the text
-            all_sentences = (
-                self.dataset[self.input1_column_name][:]
-                + self.dataset[self.input2_column_name][:]
-            )
-            len_sentences1 = len(self.dataset[self.input1_column_name])
-            embeddings = self._encode_unique_texts(
-                all_sentences,
-                model,
-                task_metadata=self.task_metadata,
-                hf_split=self.hf_split,
-                hf_subset=self.hf_subset,
-                **encode_kwargs,
-            )
-            embeddings1 = embeddings[:len_sentences1]
-            embeddings2 = embeddings[len_sentences1:]
-        else:
-            embeddings1 = model.encode(
-                create_dataloader(
-                    self.dataset,
-                    task_metadata=self.task_metadata,
-                    input_column=self.input1_column_name,
-                ),
+        logger.info("Running pair classification - Encoding samples (1/2)")
+        embeddings1 = model.encode(
+            create_dataloader(
+                self.dataset,
                 task_metadata=self.task_metadata,
-                hf_split=self.hf_split,
-                hf_subset=self.hf_subset,
+                input_column=self.input1_column_name,
                 **encode_kwargs,
-            )
-            embeddings2 = model.encode(
-                create_dataloader(
-                    self.dataset,
-                    task_metadata=self.task_metadata,
-                    input_column=self.input2_column_name,
-                ),
+            ),
+            task_metadata=self.task_metadata,
+            hf_split=self.hf_split,
+            hf_subset=self.hf_subset,
+            prompt_type=self.input1_prompt_type,
+            **encode_kwargs,
+        )
+        logger.info("Running pair classification - Encoding samples (2/2)")
+        embeddings2 = model.encode(
+            create_dataloader(
+                self.dataset,
                 task_metadata=self.task_metadata,
-                hf_split=self.hf_split,
-                hf_subset=self.hf_subset,
+                input_column=self.input2_column_name,
                 **encode_kwargs,
-            )
+            ),
+            task_metadata=self.task_metadata,
+            hf_split=self.hf_split,
+            hf_subset=self.hf_subset,
+            prompt_type=self.input2_prompt_type,
+            **encode_kwargs,
+        )
 
         logger.info("Running pair classification - Evaluating pair similarity...")
         cosine_scores = 1 - paired_cosine_distances(embeddings1, embeddings2)
@@ -156,7 +148,9 @@ class PairClassificationEvaluator(Evaluator):
         hf_subset: str,
         **encode_kwargs: Any,
     ) -> np.ndarray:
-        index_map, all_unique_texts, all_texts_indexes = {}, [], []
+        index_map = {}
+        all_unique_texts: list[str] = []
+        all_texts_indexes = []
         for text in all_texts:
             text_hash = hash(text)
             if text_hash not in index_map:
@@ -168,7 +162,7 @@ class PairClassificationEvaluator(Evaluator):
         )
         all_unique_texts_embs = np.asarray(
             model.encode(
-                _create_dataloader_from_texts(all_unique_texts),
+                _create_dataloader_from_texts(all_unique_texts, **encode_kwargs),
                 task_metadata=task_metadata,
                 hf_split=hf_split,
                 hf_subset=hf_subset,

mteb/_evaluators/retrieval_evaluator.py

@@ -1,11 +1,11 @@
 import logging
 from collections.abc import Sequence
-from typing import Any
 
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import SearchProtocol
 from mteb.types import (
     CorpusDatasetType,
+    EncodeKwargs,
     QueryDatasetType,
     RelevantDocumentsType,
     RetrievalEvaluationResult,
@@ -48,7 +48,7 @@ class RetrievalEvaluator(Evaluator):
     def __call__(  # type: ignore[override]
         self,
         search_model: SearchProtocol,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
     ) -> RetrievalOutputType:
         logger.info("Running retrieval task - Indexing corpus...")
         search_model.index(

mteb/_evaluators/retrieval_metrics.py

@@ -1,5 +1,6 @@
 import logging
 from collections import defaultdict
+from collections.abc import Mapping
 from typing import Any
 
 import numpy as np
@@ -15,7 +16,7 @@ logger = logging.getLogger(__name__)
 
 def mrr(
     qrels: RelevantDocumentsType,
-    results: dict[str, dict[str, float]],
+    results: Mapping[str, Mapping[str, float]],
     k_values: list[int],
 ) -> dict[str, list[float]]:
     mrr_metrics = defaultdict(list)
@@ -32,7 +33,7 @@ def mrr(
             doc_id for doc_id in qrels[query_id] if qrels[query_id][doc_id] > 0
         }
         for k in k_values:
-            rr = 0
+            rr = 0.0
             for rank, hit in enumerate(top_hits[query_id][0:k]):
                 if hit[0] in query_relevant_docs:
                     rr = 1.0 / (rank + 1)
@@ -45,8 +46,8 @@ def recall_cap(
     qrels: RelevantDocumentsType,
     results: dict[str, dict[str, float]],
     k_values: list[int],
-) -> dict[str, list[float]]:
-    capped_recall = defaultdict(list)
+) -> dict[str, list[float | None]]:
+    capped_recall: dict[str, list[float | None]] = defaultdict(list)
 
     k_max = max(k_values)
 
@@ -139,7 +140,7 @@ def calculate_pmrr(original_run, new_run, changed_qrels):
     changes = []
     for qid in changed_qrels.keys():
         if qid + "-og" not in original_run or qid + "-changed" not in new_run:
-            logging.warning(f"Query {qid} not found in the runs for calculating p-MRR")
+            logger.warning(f"Query {qid} not found in the runs for calculating p-MRR")
             continue
         original_qid_run = original_run[qid + "-og"]
         new_qid_run = new_run[qid + "-changed"]
@@ -188,7 +189,7 @@ def evaluate_p_mrr_change(
     Returns:
         A dictionary with the scores, including "p-MRR", "og" and "changed" keys.
     """
-    followir_scores = defaultdict(dict)
+    followir_scores: dict[str, float | dict[str, float]] = defaultdict(dict)
 
     qrels_sep = {
         "og": {k: v for k, v in qrels.items() if k.endswith("-og")},
@@ -227,7 +228,7 @@ def evaluate_p_mrr_change(
             ndcg, _map, recall, precision, naucs, avg_mrr, naucs_mrr, cv_recall, {}
         )
         for key, value in scores_dict.items():
-            followir_scores[name][key] = value
+            followir_scores[name][key] = value  # type: ignore[index]
 
     return followir_scores
 
@@ -254,8 +255,8 @@ def confidence_scores(sim_scores: list[float]) -> dict[str, float]:
     sim_scores_sorted = sorted(sim_scores)[::-1]
 
     cs_max = sim_scores_sorted[0]
-    cs_std = np.std(sim_scores)
-    cs_diff1 = None
+    cs_std = float(np.std(sim_scores))
+    cs_diff1 = 0.0
     if len(sim_scores) > 1:
         cs_diff1 = sim_scores_sorted[0] - sim_scores_sorted[1]
     elif len(sim_scores) == 1:
@@ -410,7 +411,7 @@ def make_score_dict(
     cv_recall: dict[str, float],
     task_scores: dict[str, float],
     previous_results_model_meta: dict[str, Any] | None = None,
-) -> dict[str, float]:
+) -> dict[str, Any]:
     return {
         **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()},
         **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()},
@@ -528,7 +529,7 @@ def max_over_subqueries(
 
 
 def calculate_retrieval_scores(
-    results: dict[str, dict[str, float]],
+    results: Mapping[str, Mapping[str, float]],
     qrels: RelevantDocumentsType,
     k_values: list[int],
     skip_first_result: bool = False,
@@ -576,7 +577,7 @@ def calculate_retrieval_scores(
 
 
 def evaluate_abstention(
-    results: dict[str, dict[str, float]],
+    results: Mapping[str, Mapping[str, float]],
     metric_scores: dict[str, list[float]],
 ) -> dict[str, float]:
     """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997
@@ -591,21 +592,21 @@ def evaluate_abstention(
     all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())]
     all_conf_scores = [confidence_scores(sim_scores) for sim_scores in all_sim_scores]
     conf_fcts = list(all_conf_scores[0].keys())
-    all_conf_scores = {
+    all_conf_scores_ = {
         fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts
     }
-    metric_scores = {k: np.array(v) for k, v in metric_scores.items()}
+    metric_scores_ = {k: np.array(v) for k, v in metric_scores.items()}
     naucs = {}
 
-    for metric_name, scores in metric_scores.items():
-        for fct, conf_scores in all_conf_scores.items():
+    for metric_name, scores in metric_scores_.items():
+        for fct, conf_scores in all_conf_scores_.items():
             naucs[f"nAUC_{metric_name}_{fct}"] = nauc(conf_scores, scores)
 
     return naucs
 
 
 def calculate_cv_recall(
-    results: dict[str, dict[str, float]],
+    results: Mapping[str, Mapping[str, float]],
     qrels: RelevantDocumentsType,
     k_values: list[int],
     skip_first_result: bool = False,