mteb 2.5.4__py3-none-any.whl → 2.5.5__py3-none-any.whl

mteb/benchmarks/benchmark.py

@@ -19,6 +19,7 @@ class Benchmark:
 
     Args:
         name: The name of the benchmark
+        aliases: Alternative names for the benchmark
         tasks: The tasks within the benchmark.
         description: A description of the benchmark, should include its intended goal and potentially a description of its construction
         reference: A link reference, to a source containing additional information typically to a paper, leaderboard or github.
@@ -38,6 +39,7 @@ class Benchmark:
 
     name: str
     tasks: Sequence[AbsTask]
+    aliases: Sequence[str] = field(default_factory=tuple)
     description: str | None = None
     reference: StrURL | None = None
     citation: str | None = None

mteb/benchmarks/benchmarks/benchmarks.py

@@ -18,6 +18,7 @@ MMTEB_CITATION = r"""@article{enevoldsen2025mmtebmassivemultilingualtext,
 
 MTEB_EN = Benchmark(
     name="MTEB(eng, v2)",
+    aliases=["MTEB(eng)"],
     display_name="English",
     icon="https://github.com/lipis/flag-icons/raw/refs/heads/main/flags/4x3/us.svg",
     tasks=MTEBTasks(
@@ -89,6 +90,7 @@ The original MTEB leaderboard is available under the [MTEB(eng, v1)](http://mteb
 
 MTEB_ENG_CLASSIC = Benchmark(
     name="MTEB(eng, v1)",
+    aliases=["MTEB(eng, classic)", "MTEB"],
     display_name="English Legacy",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/gb.svg",
     tasks=MTEBTasks(
@@ -185,6 +187,7 @@ We recommend that you use [MTEB(eng, v2)](http://mteb-leaderboard.hf.space/?benc
 
 MTEB_MAIN_RU = Benchmark(
     name="MTEB(rus, v1)",
+    aliases=["MTEB(rus)"],
     display_name="Russian legacy",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/ru.svg",
     tasks=MTEBTasks(
@@ -344,6 +347,7 @@ RU_SCI_BENCH = Benchmark(
 
 MTEB_RETRIEVAL_WITH_INSTRUCTIONS = Benchmark(
     name="FollowIR",
+    aliases=["MTEB(Retrieval w/Instructions)"],
     display_name="Instruction Following",
     tasks=get_tasks(
         tasks=[
@@ -394,7 +398,9 @@ MTEB_RETRIEVAL_WITH_DOMAIN_INSTRUCTIONS = Benchmark(
 )
 
 MTEB_RETRIEVAL_LAW = Benchmark(
-    name="MTEB(Law, v1)",  # This benchmark is likely in the need of an update
+    # This benchmark is likely in the need of an update
+    name="MTEB(Law, v1)",
+    aliases=["MTEB(law)"],
     display_name="Legal",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-map-library.svg",
     tasks=get_tasks(
@@ -416,6 +422,7 @@ MTEB_RETRIEVAL_LAW = Benchmark(
 
 MTEB_RETRIEVAL_MEDICAL = Benchmark(
     name="MTEB(Medical, v1)",
+    aliases=["MTEB(Medical)"],
     display_name="Medical",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-map-hospital.svg",
     tasks=get_tasks(
@@ -469,6 +476,7 @@ MTEB_MINERS_BITEXT_MINING = Benchmark(
 
 SEB = Benchmark(
     name="MTEB(Scandinavian, v1)",
+    aliases=["MTEB(Scandinavian)", "SEB"],
     display_name="Scandinavian",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/dk.svg",
     language_view=["dan-Latn", "swe-Latn", "nno-Latn", "nob-Latn"],
@@ -595,6 +603,7 @@ RAR_b = Benchmark(
 
 MTEB_FRA = Benchmark(
     name="MTEB(fra, v1)",
+    aliases=["MTEB(fra)"],
     display_name="French",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/fr.svg",
     tasks=MTEBTasks(
@@ -653,6 +662,7 @@ MTEB_FRA = Benchmark(
 
 MTEB_DEU = Benchmark(
     name="MTEB(deu, v1)",
+    aliases=["MTEB(deu)"],
     display_name="German",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/de.svg",
     tasks=get_tasks(
@@ -704,6 +714,7 @@ MTEB_DEU = Benchmark(
 
 MTEB_KOR = Benchmark(
     name="MTEB(kor, v1)",
+    aliases=["MTEB(kor)"],
     display_name="Korean",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/kr.svg",
     tasks=get_tasks(
@@ -728,6 +739,7 @@ MTEB_KOR = Benchmark(
 
 MTEB_POL = Benchmark(
     name="MTEB(pol, v1)",
+    aliases=["MTEB(pol)"],
     display_name="Polish",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/pl.svg",
     tasks=MTEBTasks(
@@ -777,6 +789,7 @@ two novel clustering tasks.""",  # Rephrased from the abstract
 
 MTEB_code = Benchmark(
     name="MTEB(Code, v1)",
+    aliases=["MTEB(code)"],
     display_name="Code",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-tech-electronics.svg",
     tasks=get_tasks(
@@ -953,6 +966,7 @@ MTEB_multilingual_v1 = Benchmark(
 
 MTEB_multilingual_v2 = Benchmark(
     name="MTEB(Multilingual, v2)",
+    aliases=["MTEB(Multilingual)", "MMTEB"],
     display_name="Multilingual",
     language_view=[
         "eng-Latn",  # English
@@ -986,6 +1000,7 @@ MTEB_multilingual_v2 = Benchmark(
 
 MTEB_JPN = Benchmark(
     name="MTEB(jpn, v1)",
+    aliases=["MTEB(jpn)"],
     display_name="Japanese Legacy",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/jp.svg",
     tasks=get_tasks(
@@ -1056,6 +1071,7 @@ indic_languages = [
 
 MTEB_INDIC = Benchmark(
     name="MTEB(Indic, v1)",
+    aliases=["MTEB(Indic)"],
     display_name="Indic",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/in.svg",
     tasks=MTEBTasks(
@@ -1146,6 +1162,7 @@ eu_languages = [
 
 MTEB_EU = Benchmark(
     name="MTEB(Europe, v1)",
+    aliases=["MTEB(Europe)"],
     display_name="European",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/eu.svg",
     tasks=get_tasks(
@@ -1285,6 +1302,7 @@ BRIGHT = Benchmark(
 
 BRIGHT_LONG = Benchmark(
     name="BRIGHT (long)",
+    aliases=["BRIGHT(long)"],
     tasks=MTEBTasks(
         (
             get_task(
@@ -1400,6 +1418,7 @@ NANOBEIR = Benchmark(
 
 C_MTEB = Benchmark(
     name="MTEB(cmn, v1)",
+    aliases=["MTEB(Chinese)", "CMTEB"],
     display_name="Chinese",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/cn.svg",
     tasks=MTEBTasks(
@@ -1466,6 +1485,7 @@ C_MTEB = Benchmark(
 
 FA_MTEB = Benchmark(
     name="MTEB(fas, v1)",
+    aliases=["FaMTEB(fas, beta)"],
     display_name="Farsi Legacy",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/ir.svg",
     tasks=get_tasks(
@@ -2347,6 +2367,7 @@ VIDORE_V3 = VidoreBenchmark(
 
 VISUAL_DOCUMENT_RETRIEVAL = VidoreBenchmark(
     name="ViDoRe(v1&v2)",
+    aliases=["VisualDocumentRetrieval"],
     display_name="ViDoRe (V1&V2)",
     tasks=get_tasks(
         tasks=[

mteb/benchmarks/get_benchmark.py

@@ -1,6 +1,5 @@
 import difflib
 import logging
-import warnings
 from functools import lru_cache
 
 from .benchmark import Benchmark
@@ -20,53 +19,16 @@ def _build_registry() -> dict[str, Benchmark]:
     return benchmark_registry
 
 
-def _get_previous_benchmark_names() -> dict[str, str]:
-    from .benchmarks import (
-        BRIGHT_LONG,
-        C_MTEB,
-        FA_MTEB,
-        MTEB_DEU,
-        MTEB_EN,
-        MTEB_ENG_CLASSIC,
-        MTEB_EU,
-        MTEB_FRA,
-        MTEB_INDIC,
-        MTEB_JPN,
-        MTEB_KOR,
-        MTEB_MAIN_RU,
-        MTEB_POL,
-        MTEB_RETRIEVAL_LAW,
-        MTEB_RETRIEVAL_MEDICAL,
-        MTEB_RETRIEVAL_WITH_INSTRUCTIONS,
-        SEB,
-        VISUAL_DOCUMENT_RETRIEVAL,
-        MTEB_code,
-        MTEB_multilingual_v2,
-    )
-
-    previous_benchmark_names = {
-        "MTEB(eng)": MTEB_EN.name,
-        "MTEB(eng, classic)": MTEB_ENG_CLASSIC.name,
-        "MTEB(rus)": MTEB_MAIN_RU.name,
-        "MTEB(Retrieval w/Instructions)": MTEB_RETRIEVAL_WITH_INSTRUCTIONS.name,
-        "MTEB(law)": MTEB_RETRIEVAL_LAW.name,
-        "MTEB(Medical)": MTEB_RETRIEVAL_MEDICAL.name,
-        "MTEB(Scandinavian)": SEB.name,
-        "MTEB(fra)": MTEB_FRA.name,
-        "MTEB(deu)": MTEB_DEU.name,
-        "MTEB(kor)": MTEB_KOR.name,
-        "MTEB(pol)": MTEB_POL.name,
-        "MTEB(code)": MTEB_code.name,
-        "MTEB(Multilingual)": MTEB_multilingual_v2.name,
-        "MTEB(jpn)": MTEB_JPN.name,
-        "MTEB(Indic)": MTEB_INDIC.name,
-        "MTEB(Europe)": MTEB_EU.name,
-        "MTEB(Chinese)": C_MTEB.name,
-        "FaMTEB(fas, beta)": FA_MTEB.name,
-        "BRIGHT(long)": BRIGHT_LONG.name,
-        "VisualDocumentRetrieval": VISUAL_DOCUMENT_RETRIEVAL.name,
-    }
-    return previous_benchmark_names
+@lru_cache
+def _build_aliases_registry() -> dict[str, Benchmark]:
+    import mteb.benchmarks.benchmarks as benchmark_module
+
+    aliases: dict[str, Benchmark] = {}
+    for _, inst in benchmark_module.__dict__.items():
+        if isinstance(inst, Benchmark) and inst.aliases is not None:
+            for alias in inst.aliases:
+                aliases[alias] = inst
+    return aliases
 
 
 def get_benchmark(
@@ -80,14 +42,11 @@ def get_benchmark(
     Returns:
         The Benchmark instance corresponding to the given name.
     """
-    previous_benchmark_names = _get_previous_benchmark_names()
     benchmark_registry = _build_registry()
-    if benchmark_name in previous_benchmark_names:
-        warnings.warn(
-            f"Using the previous benchmark name '{benchmark_name}' is deprecated. Please use '{previous_benchmark_names[benchmark_name]}' instead.",
-            DeprecationWarning,
-        )
-        benchmark_name = previous_benchmark_names[benchmark_name]
+    aliases_registry = _build_aliases_registry()
+
+    if benchmark_name in aliases_registry:
+        return aliases_registry[benchmark_name]
     if benchmark_name not in benchmark_registry:
         close_matches = difflib.get_close_matches(
             benchmark_name, benchmark_registry.keys()

mteb/cache.py

@@ -472,7 +472,7 @@ class ResultCache:
     def load_results(
         self,
         models: Sequence[str] | Iterable[ModelMeta] | None = None,
-        tasks: Sequence[str] | Iterable[AbsTask] | str | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | Benchmark | str | None = None,
         require_model_meta: bool = True,
         include_remote: bool = True,
         validate_and_filter: bool = False,
@@ -483,6 +483,7 @@ class ResultCache:
         Args:
             models: A list of model names to load the results for. If None it will load the results for all models.
             tasks: A list of task names to load the results for. If str is passed, then benchmark will be loaded.
+                If Benchmark is passed, then all tasks in the benchmark will be loaded.
                 If None it will load the results for all tasks.
             require_model_meta: If True it will ignore results that do not have a model_meta.json file. If false it attempt to
                 extract the model name and revision from the path.

mteb/cli/build_cli.py

@@ -290,17 +290,17 @@ def _create_meta(args: argparse.Namespace) -> None:
             "Output path already exists, use --overwrite to overwrite."
         )
 
+    benchmarks = None
     tasks: list[AbsTask] = []
     if tasks_names is not None:
         tasks = list(mteb.get_tasks(tasks_names))
     if benchmarks is not None:
         benchmarks = mteb.get_benchmarks(benchmarks)
-        for benchmark in benchmarks:
-            tasks.extend(benchmark.tasks)
 
     generate_model_card(
         model_name,
-        tasks if len(tasks) > 0 else None,
+        tasks,
+        benchmarks,
         existing_model_card_id_or_path=from_existing,
         results_cache=ResultCache(results_folder),
         output_path=output_path,

mteb/cli/generate_model_card.py

@@ -5,8 +5,8 @@ from pathlib import Path
 
 from huggingface_hub import ModelCard, ModelCardData, repo_exists
 
-from mteb import BenchmarkResults
 from mteb.abstasks.abstask import AbsTask
+from mteb.benchmarks.benchmark import Benchmark
 from mteb.cache import ResultCache
 
 logger = logging.getLogger(__name__)
@@ -15,11 +15,12 @@ logger = logging.getLogger(__name__)
 def generate_model_card(
     model_name: str,
     tasks: Sequence[AbsTask] | None = None,
+    benchmarks: Sequence[Benchmark] | None = None,
     existing_model_card_id_or_path: str | Path | None = None,
     results_cache: ResultCache = ResultCache(),
     output_path: Path = Path("model_card.md"),
     add_table_to_model_card: bool = False,
-    models_to_compare: list[str] | None = None,
+    models_to_compare: Sequence[str] | None = None,
     token: str | None = None,
     push_to_hub: bool = False,
 ) -> None:
@@ -28,6 +29,7 @@ def generate_model_card(
     Args:
         model_name: Name of the model.
         tasks: List of tasks to generate results for.
+        benchmarks: A Benchmark or list of benchmarks to generate results for.
         existing_model_card_id_or_path: Path or ID of an existing model card to update.
         results_cache: Instance of ResultCache to load results from.
         output_path: Path to save the generated model card.
@@ -41,8 +43,16 @@ def generate_model_card(
     if existing_model_card_id_or_path:
         existing_model_card = ModelCard.load(existing_model_card_id_or_path)
 
+    all_tasks: list[AbsTask] = []
+    if tasks is not None:
+        all_tasks.extend(tasks)
+
+    if benchmarks is not None:
+        for b in benchmarks:
+            all_tasks.extend(b.tasks)
+
     benchmark_results = results_cache.load_results(
-        [model_name], tasks, only_main_score=True
+        [model_name], all_tasks if all_tasks else None, only_main_score=True
     )
     eval_results = []
     for models_results in benchmark_results.model_results:
@@ -80,14 +90,12 @@ def generate_model_card(
             card_data=existing_model_card_data
         )
 
-    if models_to_compare:
-        benchmark_results = results_cache.load_results(
-            [model_name, *models_to_compare], tasks, only_main_score=True
-        )
-
     if add_table_to_model_card:
         existing_model_card = _add_table_to_model_card(
-            benchmark_results, existing_model_card
+            results_cache,
+            existing_model_card,
+            (model_name, *models_to_compare) if models_to_compare else (model_name,),
+            benchmarks or [],
         )
 
     if push_to_hub and existing_model_card_id_or_path:
@@ -102,14 +110,23 @@ def generate_model_card(
 
 
 def _add_table_to_model_card(
-    results: BenchmarkResults, model_card: ModelCard
+    results_cache: ResultCache,
+    model_card: ModelCard,
+    models: Sequence[str],
+    benchmarks: Sequence[Benchmark],
 ) -> ModelCard:
     original_content = model_card.content
-    results_df = results.to_dataframe()
-    results_df = results_df.set_index("task_name")
-    mteb_content = f"""
-# MTEB results
-{results_df.to_markdown()}
-"""
+    mteb_content = "# MTEB Results\n\n"
+
+    for benchmark in benchmarks:
+        mteb_content += f"## Benchmark: {benchmark.name}\n\n"
+        benchmark_results = results_cache.load_results(
+            tasks=benchmark,
+            models=models,
+            only_main_score=True,
+        )
+        df_results = benchmark_results.get_benchmark_result()
+        mteb_content += df_results.to_markdown(index=True) + "\n\n"
+
     model_card.content = original_content + "\n\n" + mteb_content
     return model_card

mteb/models/model_implementations/seed_1_6_embedding_models_1215.py

@@ -4,13 +4,15 @@ import base64
 import logging
 import os
 import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
 from io import BytesIO
 from typing import TYPE_CHECKING, Any
 
 import requests
 import torch
 from torch.utils.data import DataLoader
+from tqdm import tqdm
 
 from mteb._requires_package import requires_package
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -26,114 +28,6 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-
-def pil_to_base64(image, format="jpeg"):
-    if image is None:
-        return None
-    buffer = BytesIO()
-    image.save(buffer, format=format)
-    img_bytes = buffer.getvalue()
-    encoded_bytes = base64.b64encode(img_bytes)
-    return encoded_bytes.decode("utf-8")
-
-
-def multimodal_embedding(image_base64=None, text_content=None):
-    auth_token = os.getenv("VOLCES_AUTH_TOKEN")
-    model_name = "doubao-embedding-vision-251215"
-    api_url = "https://ark.cn-beijing.volces.com/api/v3/embeddings/multimodal"
-
-    headers = {
-        "Authorization": f"Bearer {auth_token}",
-        "x-ark-vlm1": "true",
-        "Content-Type": "application/json",
-    }
-
-    if image_base64 is not None and text_content is None:
-        inputs = []
-        for image in image_base64:
-            image_format = "jpeg"
-            image_data = f"data:image/{image_format};base64,{image}"
-            inputs.append({"type": "image_url", "image_url": {"url": image_data}})
-
-        payload = {"model": model_name, "input": inputs}
-    elif image_base64 is None and text_content is not None:
-        payload = {
-            "model": model_name,
-            "input": [
-                {"type": "text", "text": text_content},
-            ],
-        }
-    else:
-        inputs = []
-        for image in image_base64:
-            image_format = "jpeg"
-            image_data = f"data:image/{image_format};base64,{image}"
-            inputs.append({"type": "image_url", "image_url": {"url": image_data}})
-        inputs.append({"type": "text", "text": text_content})
-        payload = {"model": model_name, "input": inputs}
-
-    try:
-        response = requests.post(url=api_url, headers=headers, json=payload, timeout=10)
-
-        response.raise_for_status()
-        return response.json()
-
-    except requests.exceptions.HTTPError as http_err:
-        logger.error(f"HTTP error ({http_err.response.status_code}): {http_err}")
-    except requests.exceptions.JSONDecodeError:
-        logger.error("Error:The response is not in valid JSON format")
-    except requests.exceptions.Timeout:
-        logger.error("Error:Request timeout")
-    except Exception as e:
-        logger.error(f"Unknown error: {str(e)}")
-
-    return None
-
-
-def multi_thread_encode(sentences, batch_size=1, max_workers=8):
-    batches = []
-    for idx in range(0, len(sentences), batch_size):
-        batches.append((idx // batch_size, sentences[idx : idx + batch_size]))
-
-    n_batches = len(batches)
-    results = [None] * n_batches  # Pre-allocated result list
-    all_embeddings = []  # Final ordered embeddings
-
-    def _process_batch(batch_idx, batch_sentences):
-        sentence = batch_sentences[0]
-
-        retries = 5
-        while retries > 0:
-            try:
-                resp = multimodal_embedding(text_content=sentence)
-                embedding = torch.tensor(resp["data"]["embedding"])
-                break
-            except Exception as e:
-                time.sleep(1)
-                logger.warning(f"Retrying... {retries} retries left. Error: {str(e)}")
-                retries -= 1
-                if retries == 0:
-                    raise e
-        return batch_idx, embedding
-
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = {
-            executor.submit(_process_batch, idx, batch): idx for idx, batch in batches
-        }
-
-        for future in as_completed(futures):
-            batch_idx, embeddings = future.result()
-            results[batch_idx] = embeddings
-
-    for batch_embeddings in results:
-        all_embeddings.append(batch_embeddings)
-
-    all_embeddings = torch.stack(all_embeddings, dim=0)
-    all_embeddings = torch.nn.functional.normalize(all_embeddings, dim=-1)
-
-    return all_embeddings.float().cpu()
-
-
 doubao_embedding_training_data = (
     {
         "PawsXPairClassification",
@@ -166,25 +60,80 @@ class Seed16EmbeddingWrapper(AbsEncoder):
             "pip install mteb[ark]",
             "tiktoken",
         )
-        import tiktoken
 
         self._model_name = model_name
         self._max_tokens = 32768
         self._embed_dim = embed_dim
         self._available_embed_dims = [2048, 1024]
-        self._encoding = tiktoken.get_encoding(tokenizer_name)
 
-    def truncate_text_tokens(self, text: str) -> str:
-        """Truncate a string to have `max_tokens` according to the given encoding.
+    def pil_to_base64(self, image, format="jpeg"):
+        if image is None:
+            return None
+        buffer = BytesIO()
+        image.save(buffer, format=format)
+        img_bytes = buffer.getvalue()
+        encoded_bytes = base64.b64encode(img_bytes)
+        return encoded_bytes.decode("utf-8")
+
+    def multimodal_embedding(self, instruction, image_base64, text_content):
+        auth_token = os.getenv("VOLCES_AUTH_TOKEN")
+        model_name = "doubao-embedding-vision-251215"
+        api_url = "https://ark.cn-beijing.volces.com/api/v3/embeddings/multimodal"
+
+        headers = {
+            "Authorization": f"Bearer {auth_token}",
+            "x-ark-vlm1": "true",
+            "Content-Type": "application/json",
+        }
 
-        Args:
-            text: The input string to be truncated.
+        if text_content is not None and len(text_content) > self._max_tokens:
+            text_content = text_content[: self._max_tokens]
+
+        if image_base64 is not None and text_content is None:
+            inputs = []
+            for image in image_base64:
+                image_format = "jpeg"
+                image_data = f"data:image/{image_format};base64,{image}"
+                inputs.append({"type": "image_url", "image_url": {"url": image_data}})
+
+            payload = {"model": model_name, "input": inputs}
+        elif image_base64 is None and text_content is not None:
+            payload = {
+                "model": model_name,
+                "instruction": instruction,
+                "input": [
+                    {"type": "text", "text": text_content},
+                ],
+            }
+        else:
+            inputs = []
+            for image in image_base64:
+                image_format = "jpeg"
+                image_data = f"data:image/{image_format};base64,{image}"
+                inputs.append({"type": "image_url", "image_url": {"url": image_data}})
+            inputs.append({"type": "text", "text": text_content})
+            payload = {"model": model_name, "input": inputs}
+
+        max_retries = 3
+        retry_count = 0
+
+        while retry_count < max_retries:
+            response = requests.post(
+                url=api_url, headers=headers, json=payload, timeout=30
+            )
 
-        Returns:
-            The truncated string.
-        """
-        truncated_sentence = self._encoding.encode(text)[: self._max_tokens]
-        return self._encoding.decode(truncated_sentence)
+            if response.status_code != 200:
+                retry_count += 1
+                time.sleep(3)
+                continue
+
+            response_json = response.json()
+            return response_json
+
+        raise Exception(
+            f"Request failed with status code {response.status_code}. "
+            f"Response: {response.text}"
+        )
 
     def get_fused_embeddings(
         self,
@@ -204,59 +153,69 @@ class Seed16EmbeddingWrapper(AbsEncoder):
         if images is not None and texts is not None:
             assert len(texts) == len(images)
             batch_len = len(texts)
-            images_base64 = [pil_to_base64(image) for image in images]
+            images_base64 = [self.pil_to_base64(image) for image in images]
         elif images is None:
             batch_len = len(texts)
             images_base64 = [None for _ in range(batch_len)]
         elif texts is None:
             batch_len = len(images)
-            images_base64 = [pil_to_base64(image) for image in images]
+            images_base64 = [self.pil_to_base64(image) for image in images]
         else:
             raise ValueError("images and texts cannot be None at the same time")
 
-        outputs = []
-        for i in range(batch_len):
+        def process_item(
+            i, prompt_type, task_name, texts, images_base64, multimodal_embedding
+        ):
             if (
                 prompt_type == PromptType("query") or prompt_type is None
             ) and task_name in TASK_NAME_TO_INSTRUCTION:
                 instruction = TASK_NAME_TO_INSTRUCTION[task_name]
                 instruction = instruction.rstrip("{}").rstrip("\n")
-                if texts[i] != "":
-                    input_text = (
-                        "Target_modality:Text.\n Instruction:"
-                        + instruction
-                        + "\n Query:{}"
-                    ).format(texts[i])
-                else:
-                    input_text = (
-                        "Target_modality:Text.\n Instruction:"
-                        + instruction
-                        + "\n Query:"
-                    )
+                instruction = (
+                    "Target_modality:Text.\n Instruction:" + instruction + "\n Query:"
+                )
+                input_text = texts[i]
             else:
                 if texts[i] != "" and images_base64[i] is not None:
-                    instruction = "Instruction: Compress the the text and image into one word.\n Query: {}"
-                    input_text = instruction.format(texts[i])
+                    instruction = "Instruction: Compress the text and image into one word.\n Query:"
+                    input_text = texts[i]
                 elif texts[i] != "":
                     instruction = (
-                        "Instruction: Compress the the text into one word.\n Query: {}"
+                        "Instruction: Compress the text into one word.\n Query:"
                     )
-                    input_text = instruction.format(texts[i])
+                    input_text = texts[i]
                 elif images_base64[i] is not None:
                     instruction = (
-                        "Instruction: Compress the the image into one word.\n Query:"
+                        "Instruction: Compress the image into one word.\n Query:"
                     )
-                    input_text = instruction
+                    input_text = None
                 else:
                     raise ValueError("image and text are both None")
 
             resp = multimodal_embedding(
-                image_base64=[images_base64[i]], text_content=input_text
+                instruction=instruction,
+                image_base64=images_base64[i],
+                text_content=input_text,
             )
             embedding = torch.tensor(resp["data"]["embedding"])
             embedding = torch.reshape(embedding, (1, -1))
+            return embedding
+
+        outputs = []
+        process_partial = partial(
+            process_item,
+            prompt_type=prompt_type,
+            task_name=task_name,
+            texts=texts,
+            images_base64=images_base64,
+            multimodal_embedding=self.multimodal_embedding,
+        )
+        with ThreadPoolExecutor(max_workers=15) as executor:
+            futures = [executor.submit(process_partial, i) for i in range(batch_len)]
+            for future in tqdm(futures, total=batch_len, desc="Encoding"):
+                outputs.append(future.result())
 
-        outputs = torch.stack(outputs, dim=0)
+        outputs = torch.stack(outputs, dim=0).squeeze(1)
 
         if self._embed_dim is not None:
             outputs = outputs[:, : self._embed_dim]
@@ -273,13 +232,21 @@ class Seed16EmbeddingWrapper(AbsEncoder):
         prompt_type: PromptType | None = None,
         **kwargs: Any,
     ) -> Array:
-        sentences = [text for batch in inputs for text in batch["text"]]
-        images = [image for batch in inputs for image in batch["image"]]
+        if "text" in inputs.dataset.features:
+            sentences = [text for batch in inputs for text in batch["text"]]
+        else:
+            sentences = None
+
+        if "image" in inputs.dataset.features:
+            images = [image for batch in inputs for image in batch["image"]]
+        else:
+            images = None
 
         return self.get_fused_embeddings(
             texts=sentences,
             images=images,
             task_name=task_metadata.name,
+            prompt_type=prompt_type,
             **kwargs,
         )
 

mteb/models/model_meta.py

@@ -22,6 +22,7 @@ from huggingface_hub import (
 from huggingface_hub.errors import (
     EntryNotFoundError,
     GatedRepoError,
+    HFValidationError,
     NotASafetensorsRepoError,
     RepositoryNotFoundError,
     SafetensorsParsingError,
@@ -305,7 +306,7 @@ class ModelMeta(BaseModel):
         embedding_dim = None
         max_tokens = None
 
-        if model_name and compute_metadata and repo_exists(model_name):
+        if model_name and compute_metadata and _repo_exists(model_name):
             reference = "https://huggingface.co/" + model_name
             card = ModelCard.load(model_name)
             card_data: ModelCardData = card.data
@@ -414,7 +415,7 @@ class ModelMeta(BaseModel):
             meta.framework.append("Sentence Transformers")
         meta.modalities = ["text"]
 
-        if model and compute_metadata and repo_exists(model):
+        if model and compute_metadata and _repo_exists(model):
             # have max_seq_length field
             sbert_config = _get_json_from_hub(
                 model, "sentence_bert_config.json", "model", revision=revision
@@ -785,3 +786,19 @@ def _get_file_on_hub(
     except (GatedRepoError, RepositoryNotFoundError, EntryNotFoundError) as e:
         logger.warning(f"Can't get file {file_name} of {repo_id}: {e}")
         return None
+
+
+def _repo_exists(repo_id: str, repo_type: str | None = None) -> bool:
+    """Checks if a repository exists on HuggingFace Hub.
+
+    Repo exists will raise HFValidationError for invalid local paths
+
+    Args:
+        repo_id: The repository ID.
+        repo_type: The type of repository (e.g., "model", "dataset", "space").
+    """
+    try:
+        return repo_exists(repo_id=repo_id, repo_type=repo_type)
+    except HFValidationError as e:
+        logger.warning(f"Can't check existence of {repo_id}: {e}")
+        return False

{mteb-2.5.4.dist-info → mteb-2.5.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.5.4
+Version: 2.5.5
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.5.4.dist-info → mteb-2.5.5.dist-info}/RECORD

@@ -5,7 +5,7 @@ mteb/_helpful_enum.py,sha256=jh73N1jlcpg7RGz4bj8UpctiMNvqvHpp9wrB7SYEzIU,510
 mteb/_log_once.py,sha256=-tUKzxGQzf2LZSuQXi97oYFXMta1B6GEYXd7BPqssvY,1095
 mteb/_requires_package.py,sha256=eHg_TD9BVZRzNCcQQrUP17d8M1DF_vOd_tVx54AmAnM,3017
 mteb/_set_seed.py,sha256=HPlPRl__Pe6IG-4UgJqTfplcivJ_wA2kaClbXoHQedM,1178
-mteb/cache.py,sha256=8EB1irD7AHKxYuELFYzEC4GA04TMNThXJJSHixQZw6k,21494
+mteb/cache.py,sha256=M9UkWEqSA_Ro3_jc09k-XjVQy7amIDgeHgyO8VmHhmI,21594
 mteb/deprecated_evaluator.py,sha256=LCnM-kG2SBkh-xqVd4MurExsVMlFOIycSb7sHz2S_Cw,27634
 mteb/evaluate.py,sha256=6h06XsolgVCJEq9j6NA5ebwH2rSLsyIdtrxHanlqQfk,19185
 mteb/filter_tasks.py,sha256=D9g2o79aQiA5va7u_QKtMlZNDUmYwZGqCDpaKhBimWQ,7335
@@ -58,15 +58,15 @@ mteb/abstasks/text/reranking.py,sha256=QMgAAndGYRzvQdlhjLRMxrh_yrJZ0VQH40I-7mXo1
 mteb/abstasks/text/summarization.py,sha256=Sr-QX7T8SDS2dudSEspZHUtH_sxF_8A_tgfbkZNT3cA,7137
 mteb/benchmarks/__init__.py,sha256=MQEVeli-zLaJ7Xg0z7RhXQwsdmm7Ht_W2Ln0rZo1Szc,225
 mteb/benchmarks/_create_table.py,sha256=b2RqGqi0ZonKbHecEcZiF4pkfE96smFRIzxOI82ETA8,22304
-mteb/benchmarks/benchmark.py,sha256=MdRdGIGyYOH1_wK9O6NSAGjsA_QGjU5VKBoLPJcd4PE,5583
-mteb/benchmarks/get_benchmark.py,sha256=-n_O-gitRKZi48gJKNgGuI36hsP7yLVSiwulnMHN7Gw,3935
+mteb/benchmarks/benchmark.py,sha256=RheQOo0iQbu_ylN7oFLr2r-z_ahrMCTvKscVuwUx6yo,5694
+mteb/benchmarks/get_benchmark.py,sha256=nzR6cu5yXu1kIJKhd4A2R62xp43Z62bluPbOpNXHMWQ,2545
 mteb/benchmarks/benchmarks/__init__.py,sha256=73NYNv98q-tRCqf2YHabvElz_a8g_mF75HTup0J-E5E,2220
-mteb/benchmarks/benchmarks/benchmarks.py,sha256=_8zds06sQj41JzR6BHGWk33DZE2VGvabhBoyty5oAHk,97949
+mteb/benchmarks/benchmarks/benchmarks.py,sha256=48yX0qsPL07rr14ygT28qQrCF7MBhFdrb_d2bzRkfWA,98612
 mteb/benchmarks/benchmarks/rteb_benchmarks.py,sha256=QnCSrTTaBfcRlAQp2Nu81tgv1idMXqiM16Fp2zKJ5Ys,10607
 mteb/cli/__init__.py,sha256=v-csUr3eUZElIvrGB6QGtaIdndDfNWEe9oZchsGsJpg,64
 mteb/cli/_display_tasks.py,sha256=pWKupzak8uxEIwJZbYpZpteeVprOgVT9Wr0HYeypitQ,2206
-mteb/cli/build_cli.py,sha256=ccxmjPDLT4GqBq_son4VyfxpGTnY0w_L-4H1p9izLTk,12608
-mteb/cli/generate_model_card.py,sha256=thc0I1suK87g-ND9fsKHHxrYlnQ_7AmjWFaydLtXc6Q,4288
+mteb/cli/build_cli.py,sha256=y3Pr6wBy1Y0F1NDkHWdCCtNvEXGmtSVBJfh-nGv9TLg,12543
+mteb/cli/generate_model_card.py,sha256=wX1ApQHCbox9z8QIiHSomcbTiCIHmsoUTnsxmAv7e-g,4945
 mteb/descriptive_stats/BitextMining/BUCC.json,sha256=7zXoJaZacNdqMSG60jPZGIDJ1is_bxbVlcrVyImPRxw,3745
 mteb/descriptive_stats/BitextMining/BUCC.v2.json,sha256=IRPOKaIaUD31okNe12nQV2E1JeYK_Fo25Tz7d-utATM,3716
 mteb/descriptive_stats/BitextMining/BibleNLPBitextMining.json,sha256=BxzjiVoeXrSKaeBIVytLKMf2yx_6ksZ4GddPtTU8MWY,1248649
@@ -1448,7 +1448,7 @@ mteb/models/__init__.py,sha256=ABTuoqiBjBtBWW3LYY7ItBHdylR6jWoy06HH0g6j6fU,910
 mteb/models/abs_encoder.py,sha256=HSJTjvcPYJRsKhhZeK2r6YP241EqpovwBcAuX1NevKE,16553
 mteb/models/get_model_meta.py,sha256=76BlPX5NuoNpo223OrcjD6a15Ee23G2RRlQfQt8mrAA,5620
 mteb/models/instruct_wrapper.py,sha256=e6id0oNMQd7ulDCkB-2IGaF2JK5S3Tiwcn-QFG-ufDk,9292
-mteb/models/model_meta.py,sha256=p4Xl4Yae3kIp2k_ebV4VNGAzthQVDgDWDW8toiuV8S8,29842
+mteb/models/model_meta.py,sha256=5hWcv0RfVt3RcBJ2Vsyc2_8pGP42MGNRqXOzsIe4DJ0,30397
 mteb/models/models_protocols.py,sha256=D2hYWn_UBGMaKtRwBx3u0B0ni6lHJjSzTxX21XFNwIc,8917
 mteb/models/search_wrappers.py,sha256=yu3BnXLqE5JbOD14cF2mhyjvlF5LRKPfgk8uUuDhbjI,20939
 mteb/models/sentence_transformer_wrapper.py,sha256=3zmWttzmIvU8Uuz48sHJ4VQf1Mu_kvSLCdxezoQ6FN4,12545
@@ -1563,7 +1563,7 @@ mteb/models/model_implementations/samilpwc_models.py,sha256=ZcMUO_pWXARqzBa_2G6q
 mteb/models/model_implementations/sarashina_embedding_models.py,sha256=LmJAsZ_zXywQwpQspQRB83jThRq2Lc8wxZt8K8UYzRw,8467
 mteb/models/model_implementations/searchmap_models.py,sha256=WpwYwv0xWmig-rTGK0Li7m8ppADV-Qhy9BiHPlhGZug,1930
 mteb/models/model_implementations/seed_1_6_embedding_models.py,sha256=gcGKEY-n7DWGPlXYhO_kcNJ3lkBEnbw8NUxADNs3siM,18635
-mteb/models/model_implementations/seed_1_6_embedding_models_1215.py,sha256=O0BlsOHaxF0EEGaoas4AdzB8f-_9W9lwfoxLypexKEo,37516
+mteb/models/model_implementations/seed_1_6_embedding_models_1215.py,sha256=OoTHcDRQGOuSzf08V62EXrSEdRsXhnMv2ZN9feJWs9s,36443
 mteb/models/model_implementations/seed_models.py,sha256=9UF2AQ0Uue8DD73SjYhHn2hLxey_7Iq9ii9TkRaA3CM,14168
 mteb/models/model_implementations/sentence_transformers_models.py,sha256=_4MbkdjZ58bell8Ss0JkyCAkLzUxTLBMofnHckRtWs0,23252
 mteb/models/model_implementations/shuu_model.py,sha256=8-hoGqELHQRQ1QFhjwyuOY_8rqj_6f9vhE1Xi8OJ8aw,1162
@@ -2603,9 +2603,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=UKNokV9pu3G74MGebocU512aU_fFU9I9nPKnrG9Q0iE,1035
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=GwkBPmAr18Onu-vHtzHs0PFrhCozdOMiT13HwnWL4ZM,3961
-mteb-2.5.4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.5.4.dist-info/METADATA,sha256=72f7JHdvmwTqqUzMk8lT8m27KdorIiypPdxO6tRQROg,13990
-mteb-2.5.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.5.4.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.5.4.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.5.4.dist-info/RECORD,,
+mteb-2.5.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.5.5.dist-info/METADATA,sha256=9Ta6P3mxIJeO4otx8KyizC9A4mgWvQk5RW3sx_PvgL8,13990
+mteb-2.5.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.5.5.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.5.5.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.5.5.dist-info/RECORD,,