mteb 2.7.14__py3-none-any.whl → 2.7.16__py3-none-any.whl

mteb/models/get_model_meta.py

@@ -140,7 +140,7 @@ def get_model_meta(
         model_name: Name of the model to fetch
         revision: Revision of the model to fetch
         fetch_from_hf: Whether to fetch the model from HuggingFace Hub if not found in the registry
-        fill_missing: Computes missing attributes from the metadata including number of parameters and memory usage.
+        fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
 
     Returns:
         A model metadata object
@@ -161,7 +161,7 @@ def get_model_meta(
 
         if fill_missing and fetch_from_hf:
             original_meta_dict = model_meta.model_dump()
-            new_meta = ModelMeta.from_hub(model_name)
+            new_meta = ModelMeta.from_hub(model_name, fill_missing=fill_missing)
             new_meta_dict = new_meta.model_dump(exclude_none=True)
 
             updates = {

mteb/models/model_implementations/misc_models.py

@@ -1007,54 +1007,6 @@ thenlper__gte_small = ModelMeta(
   year={2023}
 }""",
 )
-OrlikB__KartonBERT_USE_base_v1 = ModelMeta(
-    name="OrlikB/KartonBERT-USE-base-v1",
-    model_type=["dense"],
-    revision="1f59dd58fe57995c0e867d5e29f03763eae99645",
-    release_date="2024-09-30",
-    languages=["pol-Latn"],
-    loader=sentence_transformers_loader,
-    n_parameters=103705344,
-    n_embedding_parameters=None,
-    memory_usage_mb=396,
-    max_tokens=512.0,
-    embed_dim=768,
-    license="gpl-3.0",
-    open_weights=True,
-    public_training_code=None,
-    public_training_data=None,
-    framework=["PyTorch"],
-    reference="https://huggingface.co/OrlikB/KartonBERT-USE-base-v1",
-    similarity_fn_name=ScoringFunction.COSINE,
-    use_instructions=None,
-    training_datasets=None,
-    adapted_from="KartonBERT-USE-base-v1",
-    superseded_by=None,
-)
-OrlikB__st_polish_kartonberta_base_alpha_v1 = ModelMeta(
-    name="OrlikB/st-polish-kartonberta-base-alpha-v1",
-    model_type=["dense"],
-    revision="5590a0e2d7bb43674e44d7076b3ff157f7d4a1cb",
-    release_date="2023-11-12",
-    languages=["pol-Latn"],
-    loader=sentence_transformers_loader,
-    n_parameters=None,
-    n_embedding_parameters=None,
-    memory_usage_mb=None,
-    max_tokens=514.0,
-    embed_dim=768,
-    license="lgpl",
-    open_weights=True,
-    public_training_code=None,
-    public_training_data=None,
-    framework=["PyTorch"],
-    reference="https://huggingface.co/OrlikB/st-polish-kartonberta-base-alpha-v1",
-    similarity_fn_name=ScoringFunction.COSINE,
-    use_instructions=None,
-    training_datasets=None,
-    adapted_from="st-polish-kartonberta-base-alpha-v1",
-    superseded_by=None,
-)
 sdadas__mmlw_e5_base = ModelMeta(
     name="sdadas/mmlw-e5-base",
     model_type=["dense"],

mteb/models/model_implementations/rerankers_custom.py

@@ -103,68 +103,6 @@ class BGEReranker(RerankerWrapper):
         return scores
 
 
-class MonoBERTReranker(RerankerWrapper):
-    name: str = "MonoBERT"
-
-    def __init__(
-        self,
-        model_name_or_path="castorini/monobert-large-msmarco",
-        torch_compile=False,
-        **kwargs,
-    ):
-        from transformers import AutoModelForSequenceClassification, AutoTokenizer
-
-        super().__init__(model_name_or_path, **kwargs)
-        if not self.device:
-            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        model_args = {}
-        if self.fp_options:
-            model_args["torch_dtype"] = self.fp_options
-        self.model = AutoModelForSequenceClassification.from_pretrained(
-            model_name_or_path,
-            **model_args,
-        )
-        self.model.to(self.device)
-        self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-        self.max_length = self.tokenizer.model_max_length
-        logger.info(f"Using max_length of {self.max_length}")
-
-        self.model.eval()
-
-    @torch.inference_mode()
-    def predict(
-        self,
-        inputs1: DataLoader[BatchedInput],
-        inputs2: DataLoader[BatchedInput],
-        *,
-        task_metadata: TaskMetadata,
-        hf_split: str,
-        hf_subset: str,
-        prompt_type: PromptType | None = None,
-        **kwargs: Any,
-    ) -> Array:
-        queries = [text for batch in inputs1 for text in batch["query"]]
-        instructions = None
-        if "instruction" in inputs2.dataset.features:
-            instructions = [text for batch in inputs1 for text in batch["instruction"]]
-        passages = [text for batch in inputs2 for text in batch["text"]]
-
-        if instructions is not None and instructions[0] is not None:
-            queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)]
-
-        tokens = self.tokenizer(
-            queries,
-            passages,
-            padding=True,
-            truncation="only_second",
-            return_tensors="pt",
-            max_length=self.max_length,
-        ).to(self.device)
-        output = self.model(**tokens)[0]
-        batch_scores = torch.nn.functional.log_softmax(output, dim=1)
-        return batch_scores[:, 1].exp()
-
-
 class JinaReranker(RerankerWrapper):
     name = "Jina"
 
@@ -219,31 +157,6 @@ class JinaReranker(RerankerWrapper):
         return scores
 
 
-monobert_large = ModelMeta(
-    loader=MonoBERTReranker,
-    loader_kwargs=dict(
-        fp_options="float16",
-    ),
-    name="castorini/monobert-large-msmarco",
-    model_type=["cross-encoder"],
-    languages=["eng-Latn"],
-    open_weights=True,
-    revision="0a97706f3827389da43b83348d5d18c9d53876fa",
-    release_date="2020-05-28",
-    n_parameters=None,
-    n_embedding_parameters=31_254_528,
-    memory_usage_mb=None,
-    max_tokens=None,
-    embed_dim=None,
-    license=None,
-    public_training_code=None,
-    public_training_data=None,
-    similarity_fn_name=None,
-    use_instructions=None,
-    training_datasets=None,
-    framework=["Sentence Transformers", "PyTorch", "Transformers"],
-)
-
 # languages unclear: https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual/discussions/28
 jina_reranker_multilingual = ModelMeta(
     loader=JinaReranker,

mteb/models/model_implementations/rerankers_monot5_based.py

@@ -34,7 +34,6 @@ prediction_tokens = {
     "unicamp-dl/mt5-base-en-msmarco": ["▁no", "▁yes"],
     "unicamp-dl/mt5-base-mmarco-v2": ["▁no", "▁yes"],
     "unicamp-dl/mt5-base-mmarco-v1": ["▁no", "▁yes"],
-    "unicamp-dl/mt5-13b-mmarco-100k": ["▁", "▁true"],
 }
 
 
@@ -919,28 +918,3 @@ mt5_base_mmarco_v2 = ModelMeta(
     use_instructions=None,
     framework=["PyTorch", "Transformers"],
 )
-
-mt5_13b_mmarco_100k = ModelMeta(
-    loader=MonoT5Reranker,
-    loader_kwargs=dict(
-        fp_options="float16",
-    ),
-    name="unicamp-dl/mt5-13b-mmarco-100k",
-    model_type=["cross-encoder"],
-    languages=mt5_languages,
-    open_weights=True,
-    revision="e1a4317e102a525ea9e16745ad21394a4f1bffbc",
-    release_date="2022-11-04",
-    n_parameters=None,
-    n_embedding_parameters=1_024_458_752,
-    memory_usage_mb=None,
-    max_tokens=None,
-    embed_dim=None,
-    license=None,
-    public_training_code=None,
-    public_training_data=None,
-    similarity_fn_name=None,
-    use_instructions=None,
-    training_datasets=None,
-    framework=["PyTorch", "Transformers"],
-)

mteb/models/model_meta.py

@@ -422,14 +422,16 @@ class ModelMeta(BaseModel):
         cls,
         model_name: str | None,
         revision: str | None = None,
-        compute_metadata: bool = True,
+        fill_missing: bool = True,
+        compute_metadata: bool | None = None,
     ) -> Self:
         """Generates a ModelMeta from a HuggingFace model name.
 
         Args:
             model_name: The HuggingFace model name.
             revision: Revision of the model
-            compute_metadata: Add metadata based on model card
+            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
+            compute_metadata: Deprecated. Use fill_missing instead.
 
         Returns:
             The generated ModelMeta.
@@ -437,6 +439,15 @@ class ModelMeta(BaseModel):
         loader: Callable[..., MTEBModels] | None
         model_type: MODEL_TYPES
 
+        if compute_metadata is not None:
+            warnings.warn(
+                "The compute_metadata parameter is deprecated and will be removed in a future version. "
+                f"Use fill_missing instead. Setting `fill_missing={compute_metadata}`.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            fill_missing = compute_metadata
+
         loader, model_type = cls._detect_model_type_and_loader(model_name, revision)
 
         frameworks: list[FRAMEWORKS] = ["PyTorch"]
@@ -448,7 +459,7 @@ class ModelMeta(BaseModel):
         embedding_dim = None
         max_tokens = None
 
-        if model_name and compute_metadata and _repo_exists(model_name):
+        if model_name and fill_missing and _repo_exists(model_name):
             reference = "https://huggingface.co/" + model_name
             card = ModelCard.load(model_name)
             card_data: ModelCardData = card.data
@@ -505,14 +516,16 @@ class ModelMeta(BaseModel):
         cls,
         model: SentenceTransformer,
         revision: str | None = None,
-        compute_metadata: bool = True,
+        fill_missing: bool = False,
+        compute_metadata: bool | None = None,
     ) -> Self:
         """Generates a ModelMeta from a SentenceTransformer model.
 
         Args:
             model: SentenceTransformer model.
             revision: Revision of the model
-            compute_metadata: Add metadata based on model card
+            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
+            compute_metadata: Deprecated. Use fill_missing instead.
 
         Returns:
             The generated ModelMeta.
@@ -522,7 +535,9 @@ class ModelMeta(BaseModel):
             if model.model_card_data.model_name
             else model.model_card_data.base_model
         )
-        meta = cls._from_hub(name, revision, compute_metadata)
+        meta = cls._from_hub(
+            name, revision, fill_missing=fill_missing, compute_metadata=compute_metadata
+        )
         try:
             first = model[0]
 
@@ -535,7 +550,9 @@ class ModelMeta(BaseModel):
         meta.max_tokens = model.max_seq_length
         meta.embed_dim = model.get_sentence_embedding_dimension()
         meta.similarity_fn_name = ScoringFunction.from_str(model.similarity_fn_name)
-        meta.modalities = ["text"]
+        meta.modalities = ["text"]  # best guess
+        if "Sentence Transformers" not in meta.framework:
+            meta.framework.append("Sentence Transformers")
         return meta
 
     @classmethod
@@ -543,22 +560,29 @@ class ModelMeta(BaseModel):
         cls,
         model: str,
         revision: str | None = None,
-        compute_metadata: bool = True,
+        fill_missing: bool = True,
+        compute_metadata: bool | None = None,
     ) -> Self:
         """Generates a ModelMeta for model from HuggingFace hub.
 
         Args:
             model: Name of the model from HuggingFace hub. For example, `intfloat/multilingual-e5-large`
             revision: Revision of the model
-            compute_metadata: Add metadata based on model card
+            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
+            compute_metadata: Deprecated. Use fill_missing instead.
 
         Returns:
             The generated ModelMeta.
         """
-        meta = cls._from_hub(model, revision, compute_metadata)
+        meta = cls._from_hub(
+            model,
+            revision,
+            fill_missing=fill_missing,
+            compute_metadata=compute_metadata,
+        )
         meta.modalities = ["text"]
 
-        if model and compute_metadata and _repo_exists(model):
+        if model and fill_missing and _repo_exists(model):
             # have max_seq_length field
             sbert_config = _get_json_from_hub(
                 model, "sentence_bert_config.json", "model", revision=revision
@@ -587,21 +611,28 @@ class ModelMeta(BaseModel):
         cls,
         model: CrossEncoder,
         revision: str | None = None,
-        compute_metadata: bool = True,
+        fill_missing: bool = False,
+        compute_metadata: bool | None = None,
     ) -> Self:
         """Generates a ModelMeta from a CrossEncoder.
 
         Args:
             model: The CrossEncoder model
             revision: Revision of the model
-            compute_metadata: Add metadata based on model card
+            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
+            compute_metadata: Deprecated. Use fill_missing instead.
 
         Returns:
             The generated ModelMeta
         """
         from mteb.models import CrossEncoderWrapper
 
-        meta = cls._from_hub(model.model.name_or_path, revision, compute_metadata)
+        meta = cls._from_hub(
+            model.model.name_or_path,
+            revision,
+            fill_missing=fill_missing,
+            compute_metadata=compute_metadata,
+        )
         try:
             emb = model.model.get_input_embeddings()
 
@@ -616,6 +647,8 @@ class ModelMeta(BaseModel):
         meta.embed_dim = None
         meta.modalities = ["text"]
         meta.model_type = ["cross-encoder"]
+        if "Sentence Transformers" not in meta.framework:
+            meta.framework.append("Sentence Transformers")
         return meta
 
     def is_zero_shot_on(self, tasks: Sequence[AbsTask] | Sequence[str]) -> bool | None:

{mteb-2.7.14.dist-info → mteb-2.7.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.14
+Version: 2.7.16
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.7.14.dist-info → mteb-2.7.16.dist-info}/RECORD

@@ -1479,9 +1479,9 @@ mteb/leaderboard/table.py,sha256=U5mWtrVUTk_6t8T4KAp5qlbFgKh1PD0iKICqNMfhsoY,104
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
 mteb/models/__init__.py,sha256=ABTuoqiBjBtBWW3LYY7ItBHdylR6jWoy06HH0g6j6fU,910
 mteb/models/abs_encoder.py,sha256=We9HlwWP61P4cMyZ080gywvDErA1eVsU9t46PtcNrCM,16830
-mteb/models/get_model_meta.py,sha256=WRWnVIT1n7i63BYlBRB-8BpYNtHxn7KMJOm5mzlJ8xI,7211
+mteb/models/get_model_meta.py,sha256=EUy3F2A57qfm046lDJQ2eaAaD_buHALD7XHJeYsx8vY,7234
 mteb/models/instruct_wrapper.py,sha256=XAvvbPnXiTxKhFbmusm2uS8E9BMq8QXRSzQQI1jqKzE,9781
-mteb/models/model_meta.py,sha256=E6mBB_inz9kMO8z3ixgGuB9QKWUYYzW44gSZwnY3ZbI,37316
+mteb/models/model_meta.py,sha256=B2BpzOZAxBgNwMrZ-saN0sw-Y4tjBKIbt2byDoQz3ac,38873
 mteb/models/models_protocols.py,sha256=HTB4-SYa3SeJXMMSA8o05lHTiLBbq314VW60K_PfcZY,9509
 mteb/models/search_wrappers.py,sha256=PXE1VVDWUd0LgTPJ-FxqIbGpIDWLRKo5CjrwIuu5nzw,21567
 mteb/models/sentence_transformer_wrapper.py,sha256=RsOxj-b7qzeYcxUTVJyb-lZDY4bINl4jEAEkPvKYB10,13578
@@ -1556,7 +1556,7 @@ mteb/models/model_implementations/llm2clip_models.py,sha256=X3W16uipaZ0t4Mco4lhh
 mteb/models/model_implementations/llm2vec_models.py,sha256=n86YQ8fAHU1gVtlY7tZcXq-1ab_ISxBmuk-X4MDnY4o,13348
 mteb/models/model_implementations/mcinext_models.py,sha256=T3vO9JQSmh3BICp6Y_q7j4anuA8P8LGZ4ZWnwGnF7cs,19299
 mteb/models/model_implementations/mdbr_models.py,sha256=AqsRZ-IDekIjq-FDWu0zx7Nk9ySJxaWTdRb8YhUZeu4,2828
-mteb/models/model_implementations/misc_models.py,sha256=0FkvheqPYh3JwM65F4CDlQKBDQQdjyMyfJPUdP1X2Ns,74780
+mteb/models/model_implementations/misc_models.py,sha256=JkJsyha-B5M8myLvHIwFUV14yo2lnSuBzHeO5fE9i74,73191
 mteb/models/model_implementations/mixedbread_ai_models.py,sha256=1-RD4M-16M-Rcf5CTD_R7LVoLv3cNFbmEjataQ__q94,10666
 mteb/models/model_implementations/mme5_models.py,sha256=V7BCGFkfZxkZ3ANJImvSFfP7in8OSfmkbqX-zXc_iF8,1574
 mteb/models/model_implementations/moco_models.py,sha256=6eEGpGTlI4StFRYsaNtXejhYE9GCqasUYCqB_SQy9cE,5714
@@ -1590,8 +1590,8 @@ mteb/models/model_implementations/random_baseline.py,sha256=YsITQoLbea_Iz2X84WNG
 mteb/models/model_implementations/rasgaard_models.py,sha256=_uNYP_nqJcOyoKnHNcvfJnP9gRvsv7HCWhZX2LJzQ9s,1322
 mteb/models/model_implementations/reasonir_model.py,sha256=WNWGqa9wANBL9vTdcFx51TEFXz6yHq_ygK0rij3LCL8,5217
 mteb/models/model_implementations/repllama_models.py,sha256=k6BgN2Cn41p0gQ0F1FdOTQ9OXlmFgG-2RtdvzOcCSZg,7543
-mteb/models/model_implementations/rerankers_custom.py,sha256=Bjgg_UbeHarupzzCk2rdy_Dd0_W0ZsE-DCD5v1EshnI,10953
-mteb/models/model_implementations/rerankers_monot5_based.py,sha256=6por4DPCycS8gljqKRZWUNM093bjjSVvmyQ3dzj9H6U,35321
+mteb/models/model_implementations/rerankers_custom.py,sha256=WBSA7kBRqxgb1549UwRYdtYzUovdwmW8C0PWzvGR54g,8087
+mteb/models/model_implementations/rerankers_monot5_based.py,sha256=U9ChokUEDXtkoFno-o4GeT4fXEEoFtnZn2denIafxi8,34583
 mteb/models/model_implementations/richinfoai_models.py,sha256=FsXamY-bvR5LLagtKK8fP-I5oc6B_bKp_i6_xzUYL8Y,1069
 mteb/models/model_implementations/ru_sentence_models.py,sha256=W4R985LnThJ-9XFbPnTGKb3L1QnoS3i3VXBFq94DK_w,43034
 mteb/models/model_implementations/ruri_models.py,sha256=3zYOqacB3JEnGJkMGYHqFgVkbmLo4uceJs9kzV54ivU,10819
@@ -2646,9 +2646,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=UKNokV9pu3G74MGebocU512aU_fFU9I9nPKnrG9Q0iE,1035
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=gElgSShKBXpfcqaZHhU_d2UHln1CyzUj8FN8KFun_UA,4087
-mteb-2.7.14.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.7.14.dist-info/METADATA,sha256=ZTD9D9Fuy9OCRxIXSZzh1bObP0PKSXUMqI4j3XVNR_c,14348
-mteb-2.7.14.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-mteb-2.7.14.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.7.14.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.7.14.dist-info/RECORD,,
+mteb-2.7.16.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.7.16.dist-info/METADATA,sha256=a-Rt1xa9ZgNdKf-JlM6EUZE_pKzEHoT6KGpFZUvnPo0,14348
+mteb-2.7.16.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mteb-2.7.16.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.7.16.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.7.16.dist-info/RECORD,,