mteb 2.7.13__tar.gz → 2.7.14__tar.gz

{mteb-2.7.13 → mteb-2.7.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.13
+Version: 2.7.14
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.7.13 → mteb-2.7.14}/mteb/abstasks/pair_classification.py

@@ -25,6 +25,8 @@ from mteb.types.statistics import (
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from numpy.typing import NDArray
+
     from mteb._evaluators.pair_classification_evaluator import (
         PairClassificationDistances,
     )
@@ -36,7 +38,6 @@ if TYPE_CHECKING:
         TextStatistics,
     )
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -138,7 +139,7 @@ class AbsTaskPairClassification(AbsTask):
         self, similarity_scores: PairClassificationDistances, labels: list[int]
     ) -> dict[str, float]:
         logger.info("Computing metrics...")
-        np_labels = np.asarray(labels)
+        np_labels: NDArray[np.int64] = np.asarray(labels, dtype=np.int64)
         output_scores = {}
         max_scores = defaultdict(list)
         for short_name, scores, reverse in [
@@ -281,7 +282,10 @@ class AbsTaskPairClassification(AbsTask):
         )
 
     def _compute_metrics_values(
-        self, scores: list[float], labels: np.ndarray, high_score_more_similar: bool
+        self,
+        scores: list[float],
+        labels: NDArray[np.int64],
+        high_score_more_similar: bool,
     ) -> dict[str, float]:
         """Compute the metrics for the given scores and labels.
 
@@ -315,7 +319,10 @@ class AbsTaskPairClassification(AbsTask):
         )
 
     def _find_best_acc_and_threshold(
-        self, scores: list[float], labels: np.ndarray, high_score_more_similar: bool
+        self,
+        scores: list[float],
+        labels: NDArray[np.int64],
+        high_score_more_similar: bool,
     ) -> tuple[float, float]:
         rows = list(zip(scores, labels))
         rows = sorted(rows, key=lambda x: x[0], reverse=high_score_more_similar)
@@ -323,7 +330,7 @@ class AbsTaskPairClassification(AbsTask):
         max_acc = 0
         best_threshold = -1.0
         positive_so_far = 0
-        remaining_negatives = sum(np.array(labels) == 0)
+        remaining_negatives = sum(labels == 0)
 
         for i in range(len(rows) - 1):
             score, label = rows[i]
@@ -339,10 +346,9 @@ class AbsTaskPairClassification(AbsTask):
         return max_acc, best_threshold
 
     def _find_best_f1_and_threshold(
-        self, scores, labels, high_score_more_similar: bool
+        self, scores, labels: NDArray[np.int64], high_score_more_similar: bool
     ) -> tuple[float, float, float, float]:
         scores = np.asarray(scores)
-        labels = np.asarray(labels)
 
         rows = list(zip(scores, labels))
 

{mteb-2.7.13 → mteb-2.7.14}/mteb/models/get_model_meta.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import difflib
 import logging
+import warnings
 from typing import TYPE_CHECKING, Any
 
 from mteb.models import (
@@ -122,6 +123,11 @@ def get_model(
     return model
 
 
+_MODEL_RENAMES: dict[str, str] = {
+    "bm25s": "baseline/bm25s",
+}
+
+
 def get_model_meta(
     model_name: str,
     revision: str | None = None,
@@ -139,6 +145,12 @@ def get_model_meta(
     Returns:
         A model metadata object
     """
+    if model_name in _MODEL_RENAMES:
+        new_name = _MODEL_RENAMES[model_name]
+        msg = f"The model '{model_name}' has been renamed to '{new_name}'. To prevent this warning use the new name."
+        warnings.warn(msg, DeprecationWarning, stacklevel=2)
+        model_name = new_name
+
     if model_name in MODEL_REGISTRY:
         model_meta = MODEL_REGISTRY[model_name]
 

{mteb-2.7.13 → mteb-2.7.14}/mteb/models/model_implementations/bm25.py

@@ -129,7 +129,7 @@ def bm25_loader(model_name, **kwargs) -> SearchProtocol:
 
 bm25_s = ModelMeta(
     loader=bm25_loader,
-    name="bm25s",
+    name="baseline/bm25s",
     model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,

{mteb-2.7.13 → mteb-2.7.14}/mteb/models/model_implementations/human.py

@@ -2,7 +2,7 @@ from mteb.models import ModelMeta
 
 human = ModelMeta(
     loader=None,
-    name="Human",
+    name="baseline/Human",
     model_type=["dense"],
     languages=["eng-Latn", "ara-Arab", "rus-Cyrl", "dan-Latn", "nob-Latn"],
     open_weights=True,

{mteb-2.7.13 → mteb-2.7.14}/mteb/models/model_implementations/opensearch_neural_sparse_models.py

@@ -134,7 +134,7 @@ class SparseEncoderWrapper(AbsEncoder):
 
 opensearch_neural_sparse_encoding_doc_v3_gte = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v3-gte",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="a8abaa916125ee512a7a8f4d706d07eb0128a8e6",
@@ -161,7 +161,7 @@ opensearch_neural_sparse_encoding_doc_v3_gte = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v3_distill = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v3-distill",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="babf71f3c48695e2e53a978208e8aba48335e3c0",
@@ -184,7 +184,7 @@ opensearch_neural_sparse_encoding_doc_v3_distill = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v2_distill = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v2-distill",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="8921a26c78b8559d6604eb1f5c0b74c079bee38f",
@@ -208,7 +208,7 @@ opensearch_neural_sparse_encoding_doc_v2_distill = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v2_mini = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="4af867a426867dfdd744097531046f4289a32fdd",
@@ -231,7 +231,7 @@ opensearch_neural_sparse_encoding_doc_v2_mini = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v1 = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v1",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="98cdcbd72867c547f72f2b7b7bed9cdf9f09922d",

{mteb-2.7.13 → mteb-2.7.14}/mteb/models/model_meta.py

@@ -71,7 +71,7 @@ FRAMEWORKS = Literal[
     "Transformers",
 ]
 
-MODEL_TYPES = Literal["dense", "cross-encoder", "late-interaction"]
+MODEL_TYPES = Literal["dense", "cross-encoder", "late-interaction", "sparse"]
 
 
 class ScoringFunction(HelpfulStrEnum):
@@ -266,7 +266,7 @@ class ModelMeta(BaseModel):
     @field_validator("name")
     @classmethod
     def _check_name(cls, v: str | None) -> str | None:
-        if v is None or v in ("bm25s", "Human"):
+        if v is None:
             return v
         if "/" not in v:
             raise ValueError(
@@ -302,6 +302,121 @@ class ModelMeta(BaseModel):
             raise ValueError("Model name is not set")
         return self.name.replace("/", "__").replace(" ", "_")
 
+    @classmethod
+    def _detect_cross_encoder_or_dense(
+        cls,
+        model_name: str,
+        revision: str | None,
+        sentence_transformers_loader: Callable[..., MTEBModels],
+        cross_encoder_loader: Callable[..., MTEBModels],
+    ) -> tuple[Callable[..., MTEBModels] | None, MODEL_TYPES]:
+        """Detect if model is CrossEncoder or default to dense."""
+        config = _get_json_from_hub(
+            model_name, "config.json", "model", revision=revision
+        )
+
+        if not config:
+            logger.warning(
+                f"Could not load config.json for {model_name}. "
+                "Defaulting to SentenceTransformer loader."
+            )
+            return sentence_transformers_loader, "dense"
+
+        architectures = config.get("architectures", [])
+
+        is_cross_encoder = any(
+            arch.endswith("ForSequenceClassification") for arch in architectures
+        )
+        if is_cross_encoder:
+            return cross_encoder_loader, "cross-encoder"
+
+        if cls._is_causal_lm_reranker(architectures, config, model_name):
+            return cross_encoder_loader, "cross-encoder"
+
+        logger.info(
+            f"Model {model_name} does not have modules.json or recognized architecture. "
+            "Defaulting to SentenceTransformer loader."
+        )
+        return sentence_transformers_loader, "dense"
+
+    @staticmethod
+    def _is_causal_lm_reranker(
+        architectures: list[str], config: dict[str, Any], model_name: str
+    ) -> bool:
+        """Check if model is a CausalLM-style reranker."""
+        is_causal_lm = any(arch.endswith("ForCausalLM") for arch in architectures)
+
+        if not is_causal_lm:
+            return False
+
+        num_labels = config.get("num_labels", 0)
+        model_name_lower = model_name.lower()
+
+        return (
+            num_labels > 0
+            or "rerank" in model_name_lower
+            or "cross-encoder" in model_name_lower
+        )
+
+    @classmethod
+    def _detect_model_type_and_loader(
+        cls,
+        model_name: str | None,
+        revision: str | None = None,
+    ) -> tuple[Callable[..., MTEBModels] | None, MODEL_TYPES]:
+        """Detect the model type and appropriate loader based on HuggingFace Hub configuration files.
+
+        This follows the Sentence Transformers architecture detection logic:
+        1. Check for modules.json - If present, model is a SentenceTransformer (dense encoder)
+        2. If no modules.json, check config.json for architecture:
+            - ForSequenceClassification → CrossEncoder
+            - CausalLM with reranking indicators → CrossEncoder
+        3. Default to dense (SentenceTransformer) if no clear indicators are found
+
+        Detection for CausalLM-style rerankers:
+        - Model has ForCausalLM architecture AND
+        - Has num_labels > 0 in config, OR
+        - Model name contains "rerank" or "cross-encoder"
+
+        Args:
+            model_name: The HuggingFace model name (can be None)
+            revision: The model revision
+
+        Returns:
+            A tuple of (loader_function, model_type) where:
+            - loader_function: A callable that returns MTEBModels, or None if model doesn't exist
+            - model_type: One of "dense", "cross-encoder", or "late-interaction"
+        """
+        from mteb.models import CrossEncoderWrapper, sentence_transformers_loader
+
+        if not model_name or not _repo_exists(model_name):
+            return sentence_transformers_loader, "dense"
+
+        try:
+            modules_config = _get_json_from_hub(
+                model_name, "modules.json", "model", revision=revision
+            )
+
+            if (
+                modules_config
+            ):  # SentenceTransformer/SparseEncoder (Not support for now)
+                return sentence_transformers_loader, "dense"
+            else:
+                return cls._detect_cross_encoder_or_dense(
+                    model_name,
+                    revision,
+                    sentence_transformers_loader,
+                    cross_encoder_loader=CrossEncoderWrapper,
+                )
+
+        except Exception as e:
+            logger.warning(
+                f"Error detecting model type for {model_name}: {e}. "
+                "Defaulting to SentenceTransformer loader."
+            )
+
+        return sentence_transformers_loader, "dense"
+
     @classmethod
     def _from_hub(
         cls,
@@ -319,9 +434,11 @@ class ModelMeta(BaseModel):
         Returns:
             The generated ModelMeta.
         """
-        from mteb.models import sentence_transformers_loader
+        loader: Callable[..., MTEBModels] | None
+        model_type: MODEL_TYPES
+
+        loader, model_type = cls._detect_model_type_and_loader(model_name, revision)
 
-        loader = sentence_transformers_loader
         frameworks: list[FRAMEWORKS] = ["PyTorch"]
         model_license = None
         reference = None
@@ -363,6 +480,7 @@ class ModelMeta(BaseModel):
         return cls(
             loader=loader,
             name=model_name or "no_model_name/available",
+            model_type=[model_type],
             revision=revision or "no_revision_available",
             reference=reference,
             release_date=release_date,

{mteb-2.7.13 → mteb-2.7.14}/mteb/results/model_result.py

@@ -17,6 +17,7 @@ from .task_result import TaskError, TaskResult
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable
+    from pathlib import Path
 
     from mteb.abstasks.abstask import AbsTask
     from mteb.abstasks.task_metadata import (
@@ -417,3 +418,25 @@ class ModelResult(BaseModel):
         if not mods:
             mods = self.default_modalities
         return list(set(mods))
+
+    def to_disk(self, path: Path) -> None:
+        """Save ModelResult to disk as JSON.
+
+        Args:
+            path: The path to the file to save.
+        """
+        with path.open("w") as f:
+            f.write(self.model_dump_json(indent=2))
+
+    @classmethod
+    def from_disk(cls, path: Path) -> ModelResult:
+        """Load ModelResult from disk.
+
+        Args:
+            path: The path to the JSON file to load.
+
+        Returns:
+            The loaded ModelResult object.
+        """
+        with path.open("r", encoding="utf-8") as f:
+            return cls.model_validate_json(f.read())

{mteb-2.7.13 → mteb-2.7.14}/mteb/results/task_result.py

@@ -337,16 +337,16 @@ class TaskResult(BaseModel):
             The loaded TaskResult object.
         """
         with path.open("r", encoding="utf-8") as f:
-            data = json.load(f)
+            json_str = f.read()
 
         if not load_historic_data:
             try:
-                return cls.model_validate(data)
+                return cls.model_validate_json(json_str)
             except Exception as e:
                 raise ValueError(
                     f"Error loading TaskResult from disk. You can try to load historic data by setting `load_historic_data=True`. Error: {e}"
                 )
-
+        data = json.loads(json_str)
         pre_1_11_load = (
             (
                 "mteb_version" in data
@@ -357,7 +357,7 @@ class TaskResult(BaseModel):
         )  # assume it is before 1.11.0 if the version is not present
 
         try:
-            obj: TaskResult = cls.model_validate(data)
+            obj: TaskResult = cls.model_validate_json(json_str)
         except Exception as e:
             if not pre_1_11_load:
                 raise e

{mteb-2.7.13 → mteb-2.7.14}/mteb/tasks/pair_classification/fas/fars_tail.py

@@ -1,5 +1,3 @@
-import datasets
-
 from mteb.abstasks.pair_classification import AbsTaskPairClassification
 from mteb.abstasks.task_metadata import TaskMetadata
 
@@ -8,8 +6,8 @@ class FarsTail(AbsTaskPairClassification):
     metadata = TaskMetadata(
         name="FarsTail",
         dataset={
-            "path": "azarijafari/FarsTail",
-            "revision": "7335288588f14e5a687d97fc979194c2abe6f4e7",
+            "path": "mteb/FarsTail",
+            "revision": "0fa0863dc160869b5a2d78803b4440ea3c671ff5",
         },
         description="This dataset, named FarsTail, includes 10,367 samples which are provided in both the Persian language as well as the indexed format to be useful for non-Persian researchers. The samples are generated from 3,539 multiple-choice questions with the least amount of annotator interventions in a way similar to the SciTail dataset",
         reference="https://link.springer.com/article/10.1007/s00500-023-08959-3",
@@ -37,33 +35,3 @@ class FarsTail(AbsTaskPairClassification):
 }
 """,  # after removing neutral
     )
-
-    def load_data(self, num_proc: int = 1, **kwargs) -> None:
-        if self.data_loaded:
-            return
-        path = self.metadata.dataset["path"]
-        revision = self.metadata.dataset["revision"]
-        data_files = {
-            "test": f"https://huggingface.co/datasets/{path}/resolve/{revision}/data/Test-word.csv"
-        }
-        self.dataset = datasets.load_dataset(
-            "csv", data_files=data_files, delimiter="\t"
-        )
-        self.dataset_transform()
-        self.data_loaded = True
-
-    def dataset_transform(self, num_proc: int = 1):
-        _dataset = {}
-        self.dataset = self.dataset.filter(lambda x: x["label"] != "n")
-        self.dataset = self.dataset.map(
-            lambda example: {"label": 1 if example["label"] == "e" else 0}
-        )
-        for split in self.metadata.eval_splits:
-            _dataset[split] = [
-                {
-                    "sentence1": self.dataset[split]["premise"],
-                    "sentence2": self.dataset[split]["hypothesis"],
-                    "labels": self.dataset[split]["label"],
-                }
-            ]
-        self.dataset = _dataset

{mteb-2.7.13 → mteb-2.7.14}/mteb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.13
+Version: 2.7.14
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.7.13 → mteb-2.7.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mteb"
-version = "2.7.13"
+version = "2.7.14"
 description = "Massive Text Embedding Benchmark"
 readme = "README.md"
 authors = [
@@ -428,6 +428,7 @@ extend-exclude = [
     "docs/references.bib",
     "mteb/models/model_implementations/gme_v_models.py",  # video_grid_thw `thw`
     "mteb/models/model_implementations/vista_models.py",  # self.normlized: in visual bge
+    "mteb/models/model_implementations/salesforce_models.py", # multiligual in paper title
     "tests/mock_tasks.py",  #  "denne her matche ikke den ovenstående",
     "mteb/models/model_implementations/kalm_models.py",  # prompt:  classify ist topic",
     "mteb/tasks/reranking/eng/built_bench_reranking.py",  # prompt:   descriptions from buit asset