mteb 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

mteb/descriptive_stats/Retrieval/BrightLeetcodeRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 414074,
+        "number_of_characters": 438348000,
+        "documents_text_statistics": {
+            "total_text_length": 438140779,
+            "min_text_length": 75,
+            "average_text_length": 1058.4849178125876,
+            "max_text_length": 103665,
+            "unique_texts": 413932
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 207221,
+            "min_text_length": 422,
+            "average_text_length": 1459.3028169014085,
+            "max_text_length": 3964,
+            "unique_texts": 142
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 262,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.8450704225352113,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 216
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 58744859,
+            "min_top_ranked_per_query": 412813,
+            "average_top_ranked_per_query": 413696.1901408451,
+            "max_top_ranked_per_query": 413923
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPonyLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 689,
+        "number_of_characters": 2093720,
+        "documents_text_statistics": {
+            "total_text_length": 2050155,
+            "min_text_length": 28,
+            "average_text_length": 3553.1282495667247,
+            "max_text_length": 108885,
+            "unique_texts": 577
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 43565,
+            "min_text_length": 182,
+            "average_text_length": 388.9732142857143,
+            "max_text_length": 946,
+            "unique_texts": 112
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 769,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 6.866071428571429,
+            "max_relevant_docs_per_query": 12,
+            "unique_relevant_docs": 17
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 64624,
+            "min_top_ranked_per_query": 577,
+            "average_top_ranked_per_query": 577.0,
+            "max_top_ranked_per_query": 577
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPonyRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 8006,
+        "number_of_characters": 2082980,
+        "documents_text_statistics": {
+            "total_text_length": 2039415,
+            "min_text_length": 5,
+            "average_text_length": 258.350012667849,
+            "max_text_length": 2583,
+            "unique_texts": 6183
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 43565,
+            "min_text_length": 182,
+            "average_text_length": 388.9732142857143,
+            "max_text_length": 946,
+            "unique_texts": 112
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2519,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 22.491071428571427,
+            "max_relevant_docs_per_query": 32,
+            "unique_relevant_docs": 47
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 884128,
+            "min_top_ranked_per_query": 7894,
+            "average_top_ranked_per_query": 7894.0,
+            "max_top_ranked_per_query": 7894
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPsychologyLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 613,
+        "number_of_characters": 20489389,
+        "documents_text_statistics": {
+            "total_text_length": 20419376,
+            "min_text_length": 23,
+            "average_text_length": 39881.59375,
+            "max_text_length": 669575,
+            "unique_texts": 509
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 70013,
+            "min_text_length": 166,
+            "average_text_length": 693.1980198019802,
+            "max_text_length": 2334,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 116,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1485148514851484,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 113
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 51712,
+            "min_top_ranked_per_query": 512,
+            "average_top_ranked_per_query": 512.0,
+            "max_top_ranked_per_query": 512
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPsychologyRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 52936,
+        "number_of_characters": 20372421,
+        "documents_text_statistics": {
+            "total_text_length": 20302408,
+            "min_text_length": 3,
+            "average_text_length": 384.26058483959497,
+            "max_text_length": 226941,
+            "unique_texts": 43756
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 70013,
+            "min_text_length": 166,
+            "average_text_length": 693.1980198019802,
+            "max_text_length": 2334,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 742,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 7.346534653465347,
+            "max_relevant_docs_per_query": 59,
+            "unique_relevant_docs": 738
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 5336335,
+            "min_top_ranked_per_query": 52835,
+            "average_top_ranked_per_query": 52835.0,
+            "max_top_ranked_per_query": 52835
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightRoboticsLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 609,
+        "number_of_characters": 18386897,
+        "documents_text_statistics": {
+            "total_text_length": 18166762,
+            "min_text_length": 117,
+            "average_text_length": 35761.34251968504,
+            "max_text_length": 3589928,
+            "unique_texts": 505
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 220135,
+            "min_text_length": 165,
+            "average_text_length": 2179.5544554455446,
+            "max_text_length": 19341,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 106,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0495049504950495,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 106
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 51308,
+            "min_top_ranked_per_query": 508,
+            "average_top_ranked_per_query": 508.0,
+            "max_top_ranked_per_query": 508
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightRoboticsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 62062,
+        "number_of_characters": 18167360,
+        "documents_text_statistics": {
+            "total_text_length": 17947225,
+            "min_text_length": 1,
+            "average_text_length": 289.6535724084505,
+            "max_text_length": 28637,
+            "unique_texts": 40431
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 220135,
+            "min_text_length": 165,
+            "average_text_length": 2179.5544554455446,
+            "max_text_length": 19341,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 553,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 5.475247524752476,
+            "max_relevant_docs_per_query": 36,
+            "unique_relevant_docs": 553
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 6258061,
+            "min_top_ranked_per_query": 61961,
+            "average_top_ranked_per_query": 61961.0,
+            "max_top_ranked_per_query": 61961
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightStackoverflowLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 1975,
+        "number_of_characters": 184326754,
+        "documents_text_statistics": {
+            "total_text_length": 184175475,
+            "min_text_length": 41,
+            "average_text_length": 99125.65931108719,
+            "max_text_length": 9182738,
+            "unique_texts": 1846
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 151279,
+            "min_text_length": 185,
+            "average_text_length": 1292.982905982906,
+            "max_text_length": 12432,
+            "unique_texts": 117
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 129,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1025641025641026,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 125
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 217386,
+            "min_top_ranked_per_query": 1858,
+            "average_top_ranked_per_query": 1858.0,
+            "max_top_ranked_per_query": 1858
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightStackoverflowRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 107198,
+        "number_of_characters": 183652816,
+        "documents_text_statistics": {
+            "total_text_length": 183501537,
+            "min_text_length": 1,
+            "average_text_length": 1713.6703710275399,
+            "max_text_length": 4000,
+            "unique_texts": 66270
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 151279,
+            "min_text_length": 185,
+            "average_text_length": 1292.982905982906,
+            "max_text_length": 12432,
+            "unique_texts": 117
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 819,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 7.0,
+            "max_relevant_docs_per_query": 59,
+            "unique_relevant_docs": 816
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 12528477,
+            "min_top_ranked_per_query": 107081,
+            "average_top_ranked_per_query": 107081.0,
+            "max_top_ranked_per_query": 107081
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightSustainableLivingLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 662,
+        "number_of_characters": 21154322,
+        "documents_text_statistics": {
+            "total_text_length": 21080575,
+            "min_text_length": 30,
+            "average_text_length": 38051.579422382674,
+            "max_text_length": 5732344,
+            "unique_texts": 551
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73747,
+            "min_text_length": 158,
+            "average_text_length": 682.8425925925926,
+            "max_text_length": 2843,
+            "unique_texts": 108
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 129,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1944444444444444,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 129
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 59832,
+            "min_top_ranked_per_query": 554,
+            "average_top_ranked_per_query": 554.0,
+            "max_top_ranked_per_query": 554
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightSustainableLivingRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 60900,
+        "number_of_characters": 20971763,
+        "documents_text_statistics": {
+            "total_text_length": 20898016,
+            "min_text_length": 1,
+            "average_text_length": 343.7626003421503,
+            "max_text_length": 158296,
+            "unique_texts": 50142
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73747,
+            "min_text_length": 158,
+            "average_text_length": 682.8425925925926,
+            "max_text_length": 2843,
+            "unique_texts": 108
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 604,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 5.592592592592593,
+            "max_relevant_docs_per_query": 59,
+            "unique_relevant_docs": 604
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 6565536,
+            "min_top_ranked_per_query": 60792,
+            "average_top_ranked_per_query": 60792.0,
+            "max_top_ranked_per_query": 60792
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightTheoremQAQuestionsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 188207,
+        "number_of_characters": 141817604,
+        "documents_text_statistics": {
+            "total_text_length": 141734227,
+            "min_text_length": 58,
+            "average_text_length": 753.8974425803981,
+            "max_text_length": 7334,
+            "unique_texts": 176508
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 83377,
+            "min_text_length": 12,
+            "average_text_length": 406.7170731707317,
+            "max_text_length": 1255,
+            "unique_texts": 201
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 469,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.299019607843137,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 234
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 37946536,
+            "min_top_ranked_per_query": 176970,
+            "average_top_ranked_per_query": 185105.05365853658,
+            "max_top_ranked_per_query": 188176
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightTheoremQATheoremsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 23904,
+        "number_of_characters": 20825122,
+        "documents_text_statistics": {
+            "total_text_length": 20797224,
+            "min_text_length": 74,
+            "average_text_length": 872.4033726246906,
+            "max_text_length": 19104,
+            "unique_texts": 23839
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 27898,
+            "min_text_length": 13,
+            "average_text_length": 429.2,
+            "max_text_length": 1255,
+            "unique_texts": 65
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 126,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.9384615384615385,
+            "max_relevant_docs_per_query": 6,
+            "unique_relevant_docs": 95
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 1549535,
+            "min_top_ranked_per_query": 23839,
+            "average_top_ranked_per_query": 23839.0,
+            "max_top_ranked_per_query": 23839
+        }
+    }
+}

mteb/evaluate.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 import warnings
-from collections.abc import Iterable
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, cast
@@ -17,22 +16,25 @@ from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.benchmarks.benchmark import Benchmark
 from mteb.cache import ResultCache
 from mteb.models.model_meta import ModelMeta
-from mteb.models.models_protocols import (
-    MTEBModels,
-)
 from mteb.models.sentence_transformer_wrapper import (
     CrossEncoderWrapper,
     SentenceTransformerEncoderWrapper,
 )
 from mteb.results import ModelResult, TaskResult
 from mteb.results.task_result import TaskError
-from mteb.types import HFSubset, PromptType, SplitName
-from mteb.types._encoder_io import EncodeKwargs
-from mteb.types._metadata import ModelName, Revision
+from mteb.types import PromptType
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from sentence_transformers import CrossEncoder, SentenceTransformer
 
+    from mteb.models.models_protocols import (
+        MTEBModels,
+    )
+    from mteb.types import EncodeKwargs, HFSubset, SplitName
+    from mteb.types._metadata import ModelName, Revision
+
 logger = logging.getLogger(__name__)
 
 
@@ -69,13 +71,13 @@ def _sanitize_model(
         meta = getattr(model, "mteb_model_meta")
         if not isinstance(meta, ModelMeta):
             meta = ModelMeta._from_hub(None)
-        wrapped_model = cast(MTEBModels | ModelMeta, model)
+        wrapped_model = cast("MTEBModels | ModelMeta", model)
     else:
         meta = ModelMeta._from_hub(None) if not isinstance(model, ModelMeta) else model
         wrapped_model = meta
 
-    model_name = cast(str, meta.name)
-    model_revision = cast(str, meta.revision)
+    model_name = cast("str", meta.name)
+    model_revision = cast("str", meta.revision)
 
     return wrapped_model, meta, model_name, model_revision
 
@@ -132,8 +134,8 @@ def _evaluate_task(
 
     task.check_if_dataset_is_superseded()
 
-    data_loaded = task.data_loaded
-    if not data_loaded:
+    data_preloaded = task.data_loaded
+    if not data_preloaded:
         try:
             task.load_data()
         except DatasetNotFoundError as e:
@@ -176,7 +178,7 @@ def _evaluate_task(
         kg_co2_emissions=None,
     )
 
-    if data_loaded:  # only unload if we loaded the data
+    if not data_preloaded:  # only unload if we loaded the data
         task.unload_data()
 
     return result
@@ -202,10 +204,10 @@ def _check_model_modalities(
     if isinstance(tasks, AbsTask):
         check_tasks = [tasks]
     elif isinstance(tasks, Benchmark):
-        benchmark = cast(Benchmark, tasks)
+        benchmark = cast("Benchmark", tasks)
         check_tasks = benchmark.tasks
     else:
-        check_tasks = cast(Iterable[AbsTask], tasks)
+        check_tasks = cast("Iterable[AbsTask]", tasks)
 
     warnings, errors = [], []
 
@@ -298,7 +300,7 @@ def evaluate(
                 changed.
             - "only-cache": Only load the results from the cache folder and do not run the task. Useful if you just want to load the results from the
                 cache.
-        prediction_folder: Optional folder in which to save model predictions for the task. Predictions of the tasks will be sabed in `prediction_folder/{task_name}_predictions.json`
+        prediction_folder: Optional folder in which to save model predictions for the task. Predictions of the tasks will be saved in `prediction_folder/{task_name}_predictions.json`
         show_progress_bar: Whether to show a progress bar when running the evaluation. Default is True. Setting this to False will also set the
             `encode_kwargs['show_progress_bar']` to False if encode_kwargs is unspecified.
         public_only: Run only public tasks. If None, it will attempt to run the private task.
@@ -342,7 +344,7 @@ def evaluate(
 
     # AbsTaskAggregate is a special case where we have to run multiple tasks and combine the results
     if isinstance(tasks, AbsTaskAggregate):
-        aggregated_task = cast(AbsTaskAggregate, tasks)
+        aggregated_task = cast("AbsTaskAggregate", tasks)
         results = evaluate(
             model,
             aggregated_task.metadata.tasks,
@@ -365,7 +367,7 @@ def evaluate(
     if isinstance(tasks, AbsTask):
         task = tasks
     else:
-        tasks = cast(Iterable[AbsTask], tasks)
+        tasks = cast("Iterable[AbsTask]", tasks)
         evaluate_results = []
         exceptions = []
         tasks_tqdm = tqdm(

mteb/filter_tasks.py

@@ -1,19 +1,24 @@
 """This script contains functions that are used to get an overview of the MTEB benchmark."""
 
+from __future__ import annotations
+
 import logging
-from collections.abc import Iterable, Sequence
-from typing import overload
+from typing import TYPE_CHECKING, overload
 
-from mteb.abstasks import (
-    AbsTask,
-)
 from mteb.abstasks.aggregated_task import AbsTaskAggregate
-from mteb.abstasks.task_metadata import TaskCategory, TaskDomain, TaskType
 from mteb.languages import (
     ISO_TO_LANGUAGE,
     ISO_TO_SCRIPT,
 )
-from mteb.types import Modalities
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from mteb.abstasks import (
+        AbsTask,
+    )
+    from mteb.abstasks.task_metadata import TaskCategory, TaskDomain, TaskType
+    from mteb.types import Modalities
 
 logger = logging.getLogger(__name__)
 

mteb/get_tasks.py

@@ -1,20 +1,25 @@
 """This script contains functions that are used to get an overview of the MTEB benchmark."""
 
+from __future__ import annotations
+
 import difflib
 import logging
 import warnings
 from collections import Counter, defaultdict
-from collections.abc import Iterable, Sequence
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import pandas as pd
 
 from mteb.abstasks import (
     AbsTask,
 )
-from mteb.abstasks.task_metadata import TaskCategory, TaskDomain, TaskType
 from mteb.filter_tasks import filter_tasks
-from mteb.types import Modalities
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from mteb.abstasks.task_metadata import TaskCategory, TaskDomain, TaskType
+    from mteb.types import Modalities
 
 logger = logging.getLogger(__name__)
 

mteb/languages/language_scripts.py

@@ -1,10 +1,15 @@
-from collections.abc import Iterable, Sequence
-from dataclasses import dataclass
+from __future__ import annotations
 
-from typing_extensions import Self
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
 from mteb.languages.check_language_code import check_language_code
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from typing_extensions import Self
+
 
 @dataclass
 class LanguageScripts: