mteb 2.3.6__py3-none-any.whl → 2.3.8__py3-none-any.whl

mteb/_evaluators/classification_metrics.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+
+def hamming_score(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+    """Compute the Hamming score (a.k.a. label-based accuracy) for multilabel classification.
+
+    The Hamming score is the fraction of labels that are correctly predicted for each sample,
+    averaged over all samples. For samples where both y_true and y_pred have no labels,
+    the score is 1.0 (perfect agreement).
+
+    Args:
+        y_true: Binary matrix of true labels with shape (n_samples, n_labels)
+        y_pred: Binary matrix of predicted labels with shape (n_samples, n_labels)
+
+    Returns:
+        float: Hamming score between 0.0 and 1.0
+
+    Raises:
+        ValueError: If inputs are invalid or have incompatible shapes
+        TypeError: If inputs cannot be converted to numpy arrays
+    """
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # Check shapes
+    if y_true.shape != y_pred.shape:
+        raise ValueError(
+            f"Shape mismatch: y_true {y_true.shape} != y_pred {y_pred.shape}"
+        )
+
+    # Check if arrays are empty
+    if y_true.size == 0:
+        raise ValueError("Input arrays cannot be empty")
+
+    # Ensure 2D arrays
+    if y_true.ndim != 2:
+        raise ValueError(f"Arrays must be 2D, got {y_true.ndim}D")
+
+    # Check for binary values
+    if not (np.all(np.isin(y_true, [0, 1])) and np.all(np.isin(y_pred, [0, 1]))):
+        raise ValueError("Arrays must contain only binary values (0 and 1)")
+
+    # Convert to boolean for bitwise operations
+    y_true_bool = y_true.astype(bool)
+    y_pred_bool = y_pred.astype(bool)
+
+    # Calculate intersection and union for each sample
+    intersection = (y_true_bool & y_pred_bool).sum(axis=1)
+    union = (y_true_bool | y_pred_bool).sum(axis=1)
+
+    # Handle division by zero: when union is 0, both are all zeros, so score is 1.0
+    scores = np.where(union == 0, 1.0, intersection / union)
+
+    return float(scores.mean())

mteb/abstasks/_data_filter/filters.py

@@ -0,0 +1,125 @@
+"""Simplified version of https://gist.github.com/AlexeyVatolin/ea3adc21aa7a767603ff393b22085adc from https://github.com/embeddings-benchmark/mteb/pull/2900"""
+
+import logging
+
+import datasets
+import pandas as pd
+from datasets import Dataset, DatasetDict
+
+from mteb import TaskMetadata
+
+logger = logging.getLogger(__name__)
+
+
+def deduplicate(dataset: Dataset, input_column: str) -> Dataset:
+    """Remove duplicate texts, keeping the first occurrence."""
+    unique_texts = set()
+    indices_to_keep = []
+    for i, text in enumerate(dataset[input_column]):
+        text = text.strip()
+        if text not in unique_texts:
+            unique_texts.add(text)
+            indices_to_keep.append(i)
+
+    logger.info(
+        f"[deduplicate] removed={len(dataset) - len(indices_to_keep)}/{len(dataset)}"
+    )
+    return dataset.select(indices_to_keep)
+
+
+def filter_empty(dataset: Dataset, input_column: str) -> Dataset:
+    """Filter out empty or whitespace-only examples."""
+    before = len(dataset)
+    ds = dataset.filter(lambda x: len(x[input_column].strip()) > 0)
+    logger.info(f"[filter_empty] removed={before - len(ds)}/{before}")
+    return ds
+
+
+def filter_train_leakage(
+    train_dataset: Dataset, test_dataset: Dataset, input_column: str
+) -> Dataset:
+    """Remove test examples that appear in training."""
+    train_texts = set(train_dataset[input_column])
+    before = len(test_dataset)
+    indices = [
+        i
+        for i, text in enumerate(test_dataset[input_column])
+        if text not in train_texts
+    ]
+    logger.info(f"[filter_train_leakage] removed={before - len(indices)}/{before}")
+    return test_dataset.select(indices)
+
+
+def filter_unclear_label(
+    dataset_dict: DatasetDict, input_column: str, label_column: str
+) -> DatasetDict:
+    """Remove examples where the same text appears with multiple different labels."""
+    normalized: dict[str, set[str | tuple[str, ...]]] = {}
+    logger.debug("[filter_controversial] scanning dataset for label conflicts...")
+
+    for split, ds in dataset_dict.items():
+        for text, label in zip(ds[input_column], ds[label_column]):
+            key = text.strip().lower()
+            normalized.setdefault(key, set()).add(
+                label if isinstance(label, (str, int, float)) else tuple(label)
+            )
+
+    bad_texts = {t for t, labels in normalized.items() if len(labels) > 1}
+    logger.info(f"[filter_controversial] Removing {len(bad_texts)} conflicting texts")
+
+    new_dict = {}
+    for split, ds in dataset_dict.items():
+        before = len(ds)
+        filtered = ds.filter(lambda x: x[input_column].strip().lower() not in bad_texts)
+        logger.debug(
+            f"[filter_controversial:{split}] removed={before - len(filtered)}/{before}"
+        )
+        new_dict[split] = filtered
+
+    return DatasetDict(new_dict)
+
+
+def filter_short(dataset: Dataset, input_column: str, min_words: int = 3) -> Dataset:
+    """Filter out texts with fewer than `min_words`."""
+    before = len(dataset)
+    ds = dataset.filter(lambda x: len(x[input_column].strip().split()) >= min_words)
+    logger.debug(f"[filter_short] removed={before - len(ds)}/{before}")
+    return ds
+
+
+def split_train_test(
+    ds: DatasetDict,
+    metadata: TaskMetadata,
+    train_split: str,
+    label_column: str,
+) -> DatasetDict:
+    if train_split in ds and metadata.eval_splits == train_split:
+        before = len(ds[train_split])
+        logger.info(
+            f"[split_train_test] eval_splits == train_split; performing split on {before} examples"
+        )
+        ds[train_split] = ds[train_split].cast_column(
+            label_column,
+            datasets.ClassLabel(names=list(set(ds[train_split][label_column]))),
+        )
+        label_counts = pd.Series(ds[train_split][label_column]).value_counts()
+        one_sample_labels = set(label_counts[label_counts == 1].index.tolist())
+
+        if one_sample_labels:
+            logger.info(
+                f"[split_train_test] Removing {len(one_sample_labels)} labels with only one instance"
+            )
+            ds[train_split] = ds[train_split].filter(
+                lambda x: x[label_column] not in one_sample_labels
+            )
+
+        splits = ds[train_split].train_test_split(
+            test_size=min(2048, before // 2), seed=42, stratify_by_column=label_column
+        )
+        ds = DatasetDict({train_split: splits[train_split], "test": splits["test"]})
+        metadata.eval_splits = ["test"]
+        logger.info(
+            f"[split_train_test] Train size={len(ds[train_split])}, Test size={len(ds['test'])}"
+        )
+
+    return ds

mteb/abstasks/_data_filter/task_pipelines.py

@@ -0,0 +1,102 @@
+import logging
+
+from datasets import DatasetDict
+
+from mteb import TaskMetadata
+from mteb.abstasks import AbsTaskClassification
+from mteb.abstasks._data_filter.filters import (
+    deduplicate,
+    filter_empty,
+    filter_short,
+    filter_train_leakage,
+    filter_unclear_label,
+    split_train_test,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def clean_dataset(
+    ds: DatasetDict,
+    metadata: TaskMetadata,
+    train_split: str,
+    input_column: str,
+    label_column: str,
+    subset: str | None = None,
+) -> DatasetDict:
+    """Apply the full cleaning pipeline with logging."""
+    logger.info("[clean_dataset] Starting dataset cleaning pipeline...")
+
+    transforms = [
+        ("filter_empty", filter_empty),
+        ("deduplicate", deduplicate),
+    ]
+
+    skip_cjk_codes = {"zho", "jpn", "tha", "mya", "cmn"}
+    logger.info("[clean_dataset] Applying short-text filter")
+    cur_langs = (
+        metadata.eval_langs[subset]
+        if isinstance(metadata.eval_langs, dict) and subset
+        else metadata.eval_langs
+    )
+    apply_short = not any(lang.split("-")[0] in skip_cjk_codes for lang in cur_langs)
+    if apply_short:
+        logger.info("[clean_dataset] Applying short-text filter")
+        transforms.append(("filter_short", filter_short))
+
+    for split in [train_split, *metadata.eval_splits]:
+        if split not in ds:
+            logger.warning(f"[clean_dataset] Split '{split}' missing; skipping.")
+            continue
+
+        for name, fn in transforms:
+            before = len(ds[split])
+            ds[split] = fn(ds[split], input_column=input_column)
+            logger.info(
+                f"[clean_dataset:{split}] {name} removed={before - len(ds[split])}"
+            )
+
+    ds = split_train_test(ds, metadata, train_split, label_column)
+
+    for split in metadata.eval_splits:
+        if split == train_split:
+            continue
+        before = len(ds[split])
+        ds[split] = filter_train_leakage(ds[train_split], ds[split], input_column)
+        logger.info(
+            f"[clean_dataset:{split}] leakage_removed={before - len(ds[split])}"
+        )
+
+    ds = filter_unclear_label(ds, input_column=input_column, label_column=label_column)
+
+    logger.info("[clean_dataset] Cleaning pipeline complete.")
+    return ds
+
+
+def process_classification(
+    task: AbsTaskClassification,
+) -> DatasetDict | dict[str, DatasetDict]:
+    """Process classification task dataset(s) with cleaning pipeline."""
+    if not task.data_loaded:
+        task.load_data()
+    if isinstance(task.dataset, DatasetDict):
+        return clean_dataset(
+            task.dataset,
+            task.metadata,
+            task.train_split,
+            task.input_column_name,
+            task.label_column_name,
+            subset=None,
+        )
+
+    new_ds = {}
+    for subset in task.dataset:
+        new_ds[subset] = clean_dataset(
+            task.dataset[subset],
+            task.metadata,
+            task.train_split,
+            task.input_column_name,
+            task.label_column_name,
+            subset=subset,
+        )
+    return new_ds

mteb/abstasks/multilabel_classification.py

@@ -14,6 +14,7 @@ from sklearn.preprocessing import MultiLabelBinarizer
 from typing_extensions import override
 
 from mteb._create_dataloaders import create_dataloader
+from mteb._evaluators.classification_metrics import hamming_score
 from mteb._evaluators.sklearn_evaluator import SklearnModelProtocol
 from mteb.models import EncoderProtocol
 
@@ -40,11 +41,13 @@ class MultilabelClassificationMetrics(TypedDict):
         accuracy: Accuracy of the classifier.
         lrap: Label Ranking Average Precision (LRAP) score.
         f1: Macro F1 score.
+        hamming: Hamming score (label-based accuracy).
     """
 
     accuracy: float
     lrap: float
     f1: float
+    hamming: float
 
 
 class FullMultilabelClassificationMetrics(MultilabelClassificationMetrics):
@@ -157,7 +160,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
 
         logger.info("Running multilabel classification - Evaluating classifiers...")
         all_predictions = []
-        for i_experiment, sample_indices in enumerate(train_samples):
+        for _, sample_indices in enumerate(train_samples):
             X_train = np.stack([unique_train_embeddings[idx] for idx in sample_indices])
             y_train = train_split.select(sample_indices)[self.label_column_name]
             y_train = binarizer.transform(y_train)
@@ -207,10 +210,12 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
         else:
             lrap = label_ranking_average_precision_score(y_test, y_pred)
         f1 = f1_score(y_test, y_pred, average="macro")
+        hamming = hamming_score(y_test, y_pred)
         return MultilabelClassificationMetrics(
             accuracy=accuracy,
             lrap=lrap,
             f1=f1,
+            hamming=hamming,
         )
 
     def _undersample_data_indices(
@@ -218,6 +223,8 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
     ) -> tuple[list[int], list[int]]:
         """Undersample data to have samples_per_label samples of each label.
 
+        Currently ensures that each label has at least samples_per_label samples.
+
         Returns:
             A tuple containing:
                 - List of sampled indices.

mteb/benchmarks/benchmarks/benchmarks.py

@@ -309,7 +309,7 @@ RU_SCI_BENCH = Benchmark(
     tasks=get_tasks(
         tasks=[
             # BitextMining
-            "RuSciBenchBitextMining",
+            "RuSciBenchBitextMining.v2",
             # Classification
             "RuSciBenchCoreRiscClassification",
             "RuSciBenchGRNTIClassification.v2",
@@ -963,7 +963,7 @@ MTEB_multilingual_v2 = Benchmark(
 
 MTEB_JPN = Benchmark(
     name="MTEB(jpn, v1)",
-    display_name="Japanese",
+    display_name="Japanese Legacy",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/jp.svg",
     tasks=get_tasks(
         languages=["jpn"],

mteb/descriptive_stats/BitextMining/RuSciBenchBitextMining.v2.json

@@ -0,0 +1,61 @@
+{
+    "test": {
+        "num_samples": 19928,
+        "number_of_characters": 35466331,
+        "unique_pairs": 19928,
+        "sentence1_statistics": {
+            "total_text_length": 17733346,
+            "min_text_length": 103,
+            "average_text_length": 889.8708350060217,
+            "max_text_length": 11576,
+            "unique_texts": 19928
+        },
+        "sentence2_statistics": {
+            "total_text_length": 17732985,
+            "min_text_length": 103,
+            "average_text_length": 889.8527197912485,
+            "max_text_length": 11576,
+            "unique_texts": 19928
+        },
+        "hf_subset_descriptive_stats": {
+            "ru-en": {
+                "num_samples": 9965,
+                "number_of_characters": 17734926,
+                "unique_pairs": 9965,
+                "sentence1_statistics": {
+                    "total_text_length": 8685585,
+                    "min_text_length": 103,
+                    "average_text_length": 871.6091319618665,
+                    "max_text_length": 5675,
+                    "unique_texts": 9965
+                },
+                "sentence2_statistics": {
+                    "total_text_length": 9049341,
+                    "min_text_length": 106,
+                    "average_text_length": 908.1124937280482,
+                    "max_text_length": 11576,
+                    "unique_texts": 9965
+                }
+            },
+            "en-ru": {
+                "num_samples": 9963,
+                "number_of_characters": 17731405,
+                "unique_pairs": 9963,
+                "sentence1_statistics": {
+                    "total_text_length": 9047761,
+                    "min_text_length": 106,
+                    "average_text_length": 908.1362039546322,
+                    "max_text_length": 11576,
+                    "unique_texts": 9963
+                },
+                "sentence2_statistics": {
+                    "total_text_length": 8683644,
+                    "min_text_length": 103,
+                    "average_text_length": 871.5892803372478,
+                    "max_text_length": 5675,
+                    "unique_texts": 9963
+                }
+            }
+        }
+    }
+}

mteb/leaderboard/benchmark_selector.py

@@ -71,7 +71,7 @@ GP_BENCHMARK_ENTRIES = [
                         "MTEB(cmn, v1)",
                         "MTEB(deu, v1)",
                         "MTEB(fra, v1)",
-                        "MTEB(jpn, v1)",
+                        "JMTEB(v2)",
                         "MTEB(kor, v1)",
                         "MTEB(nld, v1)",
                         "MTEB(pol, v1)",
@@ -84,7 +84,12 @@ GP_BENCHMARK_ENTRIES = [
                     MenuEntry(
                         "Other",
                         mteb.get_benchmarks(
-                            ["MTEB(eng, v1)", "MTEB(fas, v1)", "MTEB(rus, v1)"]
+                            [
+                                "MTEB(eng, v1)",
+                                "MTEB(fas, v1)",
+                                "MTEB(rus, v1)",
+                                "MTEB(jpn, v1)",
+                            ]
                         ),
                     )
                 ],

mteb/models/model_implementations/nbailab.py

@@ -0,0 +1,67 @@
+from mteb.models.model_meta import ModelMeta, ScoringFunction
+from mteb.models.sentence_transformer_wrapper import (
+    SentenceTransformerEncoderWrapper,
+)
+
+nb_sbert = ModelMeta(
+    loader=SentenceTransformerEncoderWrapper,  # type: ignore[arg-type]
+    name="NbAiLab/nb-sbert-base",
+    languages=["nno-Latn", "nob-Latn", "swe-Latn", "dan-Latn"],
+    open_weights=True,
+    revision="b95656350a076aeafd2d23763660f80655408cc6",
+    release_date="2022-11-23",
+    n_parameters=1_780_000_000,
+    memory_usage_mb=678,
+    embed_dim=4096,
+    license="apache-2.0",
+    max_tokens=75,
+    reference="https://huggingface.co/NbAiLab/nb-sbert-base",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=False,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/NbAiLab/mnli-norwegian",
+    training_datasets=set(),
+)
+
+nb_bert_large = ModelMeta(
+    loader=SentenceTransformerEncoderWrapper,  # type: ignore[arg-type]
+    name="NbAiLab/nb-bert-large",
+    languages=["nno-Latn", "nob-Latn"],
+    open_weights=True,
+    revision="f9d0fc184adab4dc354d85e1854b7634540d7550",
+    release_date="2021-04-29",
+    n_parameters=355087360,
+    memory_usage_mb=1359,
+    embed_dim=1024,
+    license="cc-by-4.0",
+    max_tokens=512,
+    reference="https://huggingface.co/NbAiLab/nb-bert-large",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=False,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/NbAiLab/nb-bert-large#training-data",
+    training_datasets=set(),
+)
+
+nb_bert_base = ModelMeta(
+    loader=SentenceTransformerEncoderWrapper,  # type: ignore[arg-type]
+    name="NbAiLab/nb-bert-base",
+    languages=["nno-Latn", "nob-Latn"],
+    open_weights=True,
+    revision="9417c3f62a3adc99f17ff92bff446f35d011f994",
+    release_date="2021-01-13",
+    n_parameters=177853440,
+    memory_usage_mb=681,
+    embed_dim=768,
+    license="cc-by-4.0",
+    max_tokens=512,
+    reference="https://huggingface.co/NbAiLab/nb-bert-base",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=False,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/NbAiLab/nb-bert-base#training-data",
+    training_datasets=set(),
+)

mteb/models/model_implementations/openai_models.py

@@ -91,10 +91,6 @@ class OpenAIModel(AbsEncoder):
 
         from openai import NotGiven
 
-        if self.model_name == "text-embedding-ada-002" and self._embed_dim is not None:
-            logger.warning(
-                "Reducing embedding size available only for text-embedding-3-* models"
-            )
         sentences = [text for batch in inputs for text in batch["text"]]
 
         mask_sents = [(i, t) for i, t in enumerate(sentences) if t.strip()]
@@ -122,13 +118,22 @@ class OpenAIModel(AbsEncoder):
 
         no_empty_embeddings = []
 
+        # Set dimensions only for models that support it
+        dimensions = (
+            self._embed_dim or NotGiven()
+            if not self.model_name == "text-embedding-ada-002"
+            else NotGiven()
+        )
+        default_kwargs = dict(
+            model=self.model_name,
+            encoding_format="float",
+            dimensions=dimensions,
+        )
+
         for sublist in tqdm(sublists, leave=False, disable=not show_progress_bar):
             try:
                 response = self._client.embeddings.create(
-                    input=sublist,
-                    model=self.model_name,
-                    encoding_format="float",
-                    dimensions=self._embed_dim or NotGiven(),
+                    input=sublist, **default_kwargs
                 )
             except Exception as e:
                 # Sleep due to too many requests
@@ -138,19 +143,13 @@ class OpenAIModel(AbsEncoder):
                 time.sleep(10)
                 try:
                     response = self._client.embeddings.create(
-                        input=sublist,
-                        model=self.model_name,
-                        encoding_format="float",
-                        dimensions=self._embed_dim or NotGiven(),
+                        input=sublist, **default_kwargs
                     )
                 except Exception as e:
                     logger.info("Sleeping for 60 seconds due to error", e)
                     time.sleep(60)
                     response = self._client.embeddings.create(
-                        input=sublist,
-                        model=self.model_name,
-                        encoding_format="float",
-                        dimensions=self._embed_dim or NotGiven(),
+                        input=sublist, **default_kwargs
                     )
             no_empty_embeddings.extend(self._to_numpy(response))
 

mteb/tasks/bitext_mining/multilingual/__init__.py

@@ -16,7 +16,7 @@ from .nusa_translation_bitext_mining import NusaTranslationBitextMining
 from .nusa_x_bitext_mining import NusaXBitextMining
 from .phinc_bitext_mining import PhincBitextMining
 from .roma_tales_bitext_mining import RomaTalesBitextMining
-from .ru_sci_bench_bitext_mining import RuSciBenchBitextMining
+from .ru_sci_bench_bitext_mining import RuSciBenchBitextMining, RuSciBenchBitextMiningV2
 from .tatoeba_bitext_mining import TatoebaBitextMining
 from .web_faq_bitext_mining import WebFAQBitextMiningQAs, WebFAQBitextMiningQuestions
 
@@ -40,6 +40,7 @@ __all__ = [
     "PhincBitextMining",
     "RomaTalesBitextMining",
     "RuSciBenchBitextMining",
+    "RuSciBenchBitextMiningV2",
     "TatoebaBitextMining",
     "WebFAQBitextMiningQAs",
     "WebFAQBitextMiningQuestions",

mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py

@@ -42,6 +42,52 @@ class RuSciBenchBitextMining(AbsTaskBitextMining):
   volume = {110},
   year = {2024},
 }
+""",
+        prompt="Given the following title and abstract of the scientific article, find its translation",
+        superseded_by="RuSciBenchBitextMining.v2",
+    )
+
+
+class RuSciBenchBitextMiningV2(AbsTaskBitextMining):
+    fast_loading = True
+    metadata = TaskMetadata(
+        name="RuSciBenchBitextMining.v2",
+        dataset={
+            "path": "mlsa-iai-msu-lab/ru_sci_bench_bitext_mining",
+            "revision": "20e815e8ac8787331546386dfd177821510f79a3",
+        },
+        description="This task focuses on finding translations of scientific articles. The dataset is sourced from eLibrary, Russia's largest electronic library of scientific publications. Russian authors often provide English translations for their abstracts and titles, and the data consists of these paired titles and abstracts. The task evaluates a model's ability to match an article's Russian title and abstract to its English counterpart, or vice versa. Compared to the previous version, 6 erroneous examples have been removed.",
+        reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
+        type="BitextMining",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs={
+            "ru-en": ["rus-Cyrl", "eng-Latn"],
+            "en-ru": ["eng-Latn", "rus-Cyrl"],
+        },
+        main_score="f1",
+        date=("2007-01-01", "2023-01-01"),
+        domains=["Academic", "Non-fiction", "Written"],
+        task_subtypes=[],
+        license="not specified",
+        dialect=[],
+        sample_creation="found",
+        annotations_creators="derived",
+        bibtex_citation=r"""
+@article{vatolin2024ruscibench,
+  author = {Vatolin, A. and Gerasimenko, N. and Ianina, A. and Vorontsov, K.},
+  doi = {10.1134/S1064562424602191},
+  issn = {1531-8362},
+  journal = {Doklady Mathematics},
+  month = {12},
+  number = {1},
+  pages = {S251--S260},
+  title = {RuSciBench: Open Benchmark for Russian and English Scientific Document Representations},
+  url = {https://doi.org/10.1134/S1064562424602191},
+  volume = {110},
+  year = {2024},
+}
 """,
         prompt="Given the following title and abstract of the scientific article, find its translation",
     )

{mteb-2.3.6.dist-info → mteb-2.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.3.6
+Version: 2.3.8
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.3.6.dist-info → mteb-2.3.8.dist-info}/RECORD

@@ -15,6 +15,7 @@ mteb/similarity_functions.py,sha256=ySSnrKl4cSKOWfyIKQPVTJtxuy2ZNfcv0COXDp22QlQ,
 mteb/_evaluators/__init__.py,sha256=Ag1_RWpxBGMpujzd3FZjI40gY_KQKIpY31tJPuk-hFg,1013
 mteb/_evaluators/_download.py,sha256=jntlcURbJxcxUjTmn2D9Tu6ZnWgDc9t5bY8p9CZCqv4,586
 mteb/_evaluators/any_sts_evaluator.py,sha256=f0V3NDP5Bfp8qEeBwP8E-Enj5F5NbFze-kGmzlkObQA,3762
+mteb/_evaluators/classification_metrics.py,sha256=TI-cMPWrIpMqpsNhhwSBY4bZUu2yM469fbcu44zolW0,1926
 mteb/_evaluators/clustering_evaluator.py,sha256=5XoKHl5LcG9jQ9oBzNAWYVpZWWUxrars3t7TdIV7xS0,2052
 mteb/_evaluators/evaluator.py,sha256=gwaeftcAKoGcIQs8jIaafynbcYrYErj6AitHBxgjn2w,807
 mteb/_evaluators/pair_classification_evaluator.py,sha256=6lgDI9wRfEK937YTS4l0W1OL1IQpHYZ4l34-Lxi9KdA,6401
@@ -37,7 +38,7 @@ mteb/abstasks/classification.py,sha256=Es9pmRdjv6xbc-KnGqVdO6dR1cc7yAHhekCZES7n5
 mteb/abstasks/clustering.py,sha256=4KcaU8_sNLmLvMhwDpNmcY2nD3BNyx_LcM-ddSv-wtY,14410
 mteb/abstasks/clustering_legacy.py,sha256=HZY8zgBgqqs5urF_to9wzqm3MnjFivs59hU6P3NrzcI,8684
 mteb/abstasks/dataset_card_template.md,sha256=aD6l8qc3_jxwoIGJNYLzse-jpRa8hu92AxpnUtNgges,5122
-mteb/abstasks/multilabel_classification.py,sha256=feLlpSKoe3b_Sb58N-9cx_5hzti1a2iA8QxcSBWSfjE,8922
+mteb/abstasks/multilabel_classification.py,sha256=rpIwI3jV2YKtmXlFS2_Ytg4yYjdjPy0q5OU4MsRJFqo,9211
 mteb/abstasks/pair_classification.py,sha256=ToOBFDiokZOz9ea-klMLj_37slbVFR3lSuihP81x9Lc,13263
 mteb/abstasks/regression.py,sha256=SeacOErZUXGLGOcwqAvht6BlbD8fcsn9QhNiFIuJGyc,8832
 mteb/abstasks/retrieval.py,sha256=7QTKYlGaGvF1lOQkB_B4qj8Vm2FxxFXNVTHhfwZO8Bw,26439
@@ -45,6 +46,9 @@ mteb/abstasks/retrieval_dataset_loaders.py,sha256=WukcFAn54rUpXULCG43eysHozXHAxo
 mteb/abstasks/sts.py,sha256=aKTivjvDtAaoYb1hz1NBv2o3UpDR-3AaeHgkDFHMBGI,9077
 mteb/abstasks/task_metadata.py,sha256=7CzYK1y-vwLUiWaEGPgU3HiolpW3UCul8Y2KJ-WSpeE,26892
 mteb/abstasks/zeroshot_classification.py,sha256=4UxBIZ1e1iRK8PRAhCWnnSDirK2vi5-w2N5ZomCnaIM,5882
+mteb/abstasks/_data_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mteb/abstasks/_data_filter/filters.py,sha256=p1QLy7V9jYVFicef61fwzCpbSpTA6rOv8CxkwEUTMvc,4585
+mteb/abstasks/_data_filter/task_pipelines.py,sha256=L56nKTGwLH3QqmzkO-Wx4Vi5vfovnnKIDN_f3M8FSiA,3078
 mteb/abstasks/image/__init__.py,sha256=NgvMJnp1g2mUv27RL-TvzA7s1BOdMG-EB1CrZfdbWdg,136
 mteb/abstasks/image/image_text_pair_classification.py,sha256=SejETTXc3g2VSWYafTe-VAHZcNpX98bgzsWsqQisIzI,7712
 mteb/abstasks/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,7 +60,7 @@ mteb/benchmarks/_create_table.py,sha256=OAiR44ynJ2fMzoBmVITQtOTYQzxIu9KUdS_HzlBl
 mteb/benchmarks/benchmark.py,sha256=70RlMyyg_wkWTlU_IbfLl-KaqRWXGCKTd8fWe9X-AQE,4173
 mteb/benchmarks/get_benchmark.py,sha256=-n_O-gitRKZi48gJKNgGuI36hsP7yLVSiwulnMHN7Gw,3935
 mteb/benchmarks/benchmarks/__init__.py,sha256=Ig5dSFunzI-F-OamruuKJVSstbG3xQNkXCxRY3Bj_Ck,2180
-mteb/benchmarks/benchmarks/benchmarks.py,sha256=qHHmJfisT75VRVoZfPcHhShCG0jY6vSWZEx-D01XxKU,94757
+mteb/benchmarks/benchmarks/benchmarks.py,sha256=vWX6QZgqF9iKAE1tIQwaXw9f8q_WiBtdgo8yj4_CHFI,94767
 mteb/benchmarks/benchmarks/rteb_benchmarks.py,sha256=QnCSrTTaBfcRlAQp2Nu81tgv1idMXqiM16Fp2zKJ5Ys,10607
 mteb/cli/__init__.py,sha256=v-csUr3eUZElIvrGB6QGtaIdndDfNWEe9oZchsGsJpg,64
 mteb/cli/_display_tasks.py,sha256=7A06dT9sSoTz6shyMvskPxuc5eHY_H7PGPlROzMP0yw,2196
@@ -83,6 +87,7 @@ mteb/descriptive_stats/BitextMining/PhincBitextMining.json,sha256=bC31IS_N3-eehB
 mteb/descriptive_stats/BitextMining/PubChemSMILESBitextMining.json,sha256=v4OzVwYV3Q-J3VitfK8zX_t2ZOZzvD4WtPeJoFeVNgI,3737
 mteb/descriptive_stats/BitextMining/RomaTalesBitextMining.json,sha256=ciHSHuKgi4Ip0eH4f6G52w4MQ0BFvjizBN1Mh-2tPNE,1415
 mteb/descriptive_stats/BitextMining/RuSciBenchBitextMining.json,sha256=f_lS0ua_dtwhmw-zqqZGGPJ4b_4u82VDicM8a71SId8,2209
+mteb/descriptive_stats/BitextMining/RuSciBenchBitextMining.v2.json,sha256=jm6tsOyOooX64kgPl_rLTv_aiHy34Nc1MudlqXgrSpQ,2214
 mteb/descriptive_stats/BitextMining/SAMSumFa.json,sha256=A1o7RjIwoNwjxRoMS6Qmn-4VGy5vX_QDK4sQAjJCM5Y,613
 mteb/descriptive_stats/BitextMining/SRNCorpusBitextMining.json,sha256=41u--q1IXxBXqFb8f_BXYdv9fIlUkbfSPldzwCNbo-w,2136
 mteb/descriptive_stats/BitextMining/SynPerChatbotRAGSumSRetrieval.json,sha256=nNEK8VHlRxDRmMjwuBA4U5RuXNZwxWyCHFCJHQbqIAQ,612
@@ -1426,7 +1431,7 @@ mteb/languages/language_scripts.py,sha256=5wix9HTYolNIpTiS5oXf2pGJyL7ftdGKs_m432
 mteb/languages/programming_languages.py,sha256=zxAakT3OSUnAuTnQ34VyeFIECnNXMlleZmAake6jsZE,211
 mteb/leaderboard/__init__.py,sha256=991roXmtRwEQysV-37hWEzWpkvPgMCGRqZTHR-hm2io,88
 mteb/leaderboard/app.py,sha256=29MxFLKEVT-roULHG5boHmsQVhld1rDGNS94r7MWlz8,33118
-mteb/leaderboard/benchmark_selector.py,sha256=uH66SI0iT1J4_fnebViWa83dQwhPi7toBv7PRL_epDw,7784
+mteb/leaderboard/benchmark_selector.py,sha256=qd-2L20RQ4ACke01UlytkhZok1dkWgfUlXzfET52kGc,7956
 mteb/leaderboard/figures.py,sha256=mPO0go_23QEhAm1RJdLiBxPFCoUiA74_ztyl6yimc7k,7553
 mteb/leaderboard/table.py,sha256=6SnrYC5GcBlvVSO6vOk6ObuqtoveBLv3JUuXqdKueG8,8333
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
@@ -1510,13 +1515,13 @@ mteb/models/model_implementations/moco_models.py,sha256=Kl0nBsqkG3crYoo5YulFq1fv
 mteb/models/model_implementations/model2vec_models.py,sha256=D-EY-6P-cKKunbgzk4DHzJL1ogpWYFhpHbTLb8qQjJw,13765
 mteb/models/model_implementations/moka_models.py,sha256=Y5do7Z4JyGxabYrjHhkBLqCKTQKotniS-f4kOgXJjag,4995
 mteb/models/model_implementations/mxbai_models.py,sha256=33ta2BnhvKYBUgE89wFgPNf-CnOb7ooumZvqHOvbZsA,3593
-mteb/models/model_implementations/nb_sbert.py,sha256=dF3WBn6ERIK7Oqp-tXdLn11Gf0Z7RKLhAoCq0YHxEug,861
+mteb/models/model_implementations/nbailab.py,sha256=bqqR0qs10IH2g5HC6K962tDMBciw8qFsNVHADNS72jk,2396
 mteb/models/model_implementations/no_instruct_sentence_models.py,sha256=6i-xbLRRNKuDpU-hwklwdQjgu1wnz5CecLSoc6kyd7Q,3976
 mteb/models/model_implementations/nomic_models.py,sha256=mT-v5Gs5-sRH8-ziCw_CtxB9ox3C6FtwWJjNghNrunw,11334
 mteb/models/model_implementations/nomic_models_vision.py,sha256=gEEieMThvw4p-QhRH0G_9-WWTvj-jqOlgFsh6O07dbc,6731
 mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py,sha256=14XSv7wGsitu0cF8P3A951gel_Py7PrKlRixkLS4qG4,6203
 mteb/models/model_implementations/nvidia_models.py,sha256=acVverAt77lURkILCVkCdXsWgY1BJoG1-ugB7yIhlIM,21555
-mteb/models/model_implementations/openai_models.py,sha256=2tJyEapIW-GtB3ZOXIHwGjSZGgJl2daE_UsbzH4NhBM,9620
+mteb/models/model_implementations/openai_models.py,sha256=loU6JByNUwRidq7lmcu8iGOtUQvzejw6HVLaF_IKCR0,9352
 mteb/models/model_implementations/openclip_models.py,sha256=W8XcokgLU1nSmMaWpYXkWWizVd3sQezcP02YtF2fXpo,11436
 mteb/models/model_implementations/opensearch_neural_sparse_models.py,sha256=fuxIjOx_kPoDps5C7LW3JllG-AZj4ktqeTNgJESHZh4,8351
 mteb/models/model_implementations/ops_moa_models.py,sha256=luWw1j2iTMx1z1ydLCjvCI89E9Yvge7ruEawivJTmfE,2413
@@ -1596,7 +1601,7 @@ mteb/tasks/bitext_mining/fas/__init__.py,sha256=srw2I-yfcLgIkRzJv-p_qZ9fg0cCKr-D
 mteb/tasks/bitext_mining/fas/fa_mteb_summary_retrieval.py,sha256=yj8l1K3CIdESwl4sl2M4HhyOrZZYQZb_EDo-EgCAJdg,3330
 mteb/tasks/bitext_mining/kat/__init__.py,sha256=a-KcFJ3Ol7R8yq02RcGjaOxEfqnwJeo7AAib-RU-JFw,116
 mteb/tasks/bitext_mining/kat/tbilisi_city_hall_bitext_mining.py,sha256=xVCxpJr7UW2KadNdn7Gsw-wZ65uz5vhRDhQZ7eILokQ,1918
-mteb/tasks/bitext_mining/multilingual/__init__.py,sha256=qDgixbHEvV3xz6JN3kFQDnvtSL-yVjl-Z8inUwimI6I,1954
+mteb/tasks/bitext_mining/multilingual/__init__.py,sha256=LcPygeOuvrka67aDkktT-2lSqcxpWPSMmd_BaxIsl24,2012
 mteb/tasks/bitext_mining/multilingual/bible_nlp_bitext_mining.py,sha256=lNbCz3dN9o3F04Y7vtNBhF-lPUNyVbAOKgUR-QKZn_8,29082
 mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py,sha256=tTKvS-v7d9V_zymCn_ZonUKlo9NI7vTyppxS9iAu8I0,2873
 mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining_fast.py,sha256=P_UHMWh2gKG6CloXmP5J2kjrCTQwoJAU1MKdLl6JFKc,1836
@@ -1615,7 +1620,7 @@ mteb/tasks/bitext_mining/multilingual/nusa_translation_bitext_mining.py,sha256=e
 mteb/tasks/bitext_mining/multilingual/nusa_x_bitext_mining.py,sha256=BphnEDdG1-IsCklJWRCs2yK7I1zVuPh7PQrrYAI540c,2309
 mteb/tasks/bitext_mining/multilingual/phinc_bitext_mining.py,sha256=53xLXpgIDk55JfCoe3pa93T_9T9sfRJAryBVcWZx5co,1477
 mteb/tasks/bitext_mining/multilingual/roma_tales_bitext_mining.py,sha256=5uwf4vhud5bQuPAcufWGcA7UBmp5YPKsyvc5OUYgK-M,1730
-mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py,sha256=lxaWPFLZpWwAy6CIGAENkBvH7iVLShE4sGf0eT0-Mac,2075
+mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py,sha256=09e8XDy-zvqpvGMuj8AIbYUGmrggPi6XvOZi9Fbw0G4,4162
 mteb/tasks/bitext_mining/multilingual/tatoeba_bitext_mining.py,sha256=Y6QnjbmL4fIuTgK8tuZfNnWmM-GwBVaAjUiAtTSgPqM,5898
 mteb/tasks/bitext_mining/multilingual/web_faq_bitext_mining.py,sha256=vWoRmAJKR0ed8mUKDqLUMe_IpFjj7Xlsw0NFhcIOJOc,14978
 mteb/tasks/bitext_mining/srn/__init__.py,sha256=XaMVvUagmgLUG6tZw2jo6fMKiVTfQpaaWZGQZo-1YYk,97
@@ -2573,9 +2578,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=CRAUc5IvqI3_9SyXDwv-PWLCXwXdZem9RePeYESRtuw,996
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=YwJsxTf1eaCI_RE-J37a-gK5wDeGAsmkeZKoZCFihSo,3755
-mteb-2.3.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.3.6.dist-info/METADATA,sha256=urz0_67bNhVt17rvN3pZdvMFt_mvxI7MFvamWkNoNjM,13923
-mteb-2.3.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.3.6.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.3.6.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.3.6.dist-info/RECORD,,
+mteb-2.3.8.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.3.8.dist-info/METADATA,sha256=QMpRmhMLXi45L0d29kCoNcEugCwDl8IWCc3wE_r-fb4,13923
+mteb-2.3.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.3.8.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.3.8.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.3.8.dist-info/RECORD,,

mteb/models/model_implementations/nb_sbert.py

@@ -1,25 +0,0 @@
-from mteb.models.model_meta import ModelMeta
-from mteb.models.sentence_transformer_wrapper import (
-    SentenceTransformerEncoderWrapper,
-)
-
-nb_sbert = ModelMeta(
-    loader=SentenceTransformerEncoderWrapper,
-    name="NbAiLab/nb-sbert-base",
-    languages=["nno-Latn", "nob-Latn", "swe-Latn", "dan-Latn"],
-    open_weights=True,
-    revision="b95656350a076aeafd2d23763660f80655408cc6",
-    release_date="2022-11-23",
-    n_parameters=1_780_000_000,
-    memory_usage_mb=678,
-    embed_dim=4096,
-    license="apache-2.0",
-    max_tokens=75,
-    reference="https://huggingface.co/NbAiLab/nb-sbert-base",
-    similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
-    use_instructions=False,
-    public_training_code=None,
-    public_training_data="https://huggingface.co/datasets/NbAiLab/mnli-norwegian",
-    training_datasets=set(),
-)