mteb 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

mteb/models/sentence_transformer_wrapper.py

@@ -7,19 +7,20 @@ from typing import TYPE_CHECKING, Any
 import numpy as np
 import torch
 from packaging.version import Version
-from torch.utils.data import DataLoader
-from typing_extensions import Unpack
 
 from mteb._log_once import LogOnce
 from mteb.models import ModelMeta
-from mteb.types import Array, BatchedInput, EncodeKwargs, PromptType
+from mteb.types import PromptType
 
 from .abs_encoder import AbsEncoder
 
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder, SentenceTransformer
+    from torch.utils.data import DataLoader
+    from typing_extensions import Unpack
 
     from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, EncodeKwargs
 
 logger = logging.getLogger(__name__)
 

mteb/models/vllm_wrapper.py

@@ -4,23 +4,25 @@ import atexit
 import gc
 import logging
 import os
-from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, Literal
 
 import numpy as np
 import torch
-from torch.utils.data import DataLoader
 
 from mteb._requires_package import requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import ModelMeta
 from mteb.models.abs_encoder import AbsEncoder
-from mteb.types import Array, BatchedInput, PromptType
+from mteb.types import PromptType
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from torch.utils.data import DataLoader
     from vllm.config import PoolerConfig  # type: ignore[import-not-found]
-else:
-    PoolerConfig = dict[str, Any]
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput
+
 
 logger = logging.getLogger(__name__)
 

mteb/results/benchmark_results.py

@@ -4,34 +4,39 @@ import functools
 import json
 import logging
 import warnings
-from collections.abc import Callable, Iterable, Iterator
 from pathlib import Path
-from typing import Any, Literal, cast
+from typing import TYPE_CHECKING, Any, Literal, cast
 
 import pandas as pd
 from packaging.version import InvalidVersion, Version
 from pydantic import BaseModel, ConfigDict
-from typing_extensions import Self
 
-from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.task_metadata import (
-    TaskDomain,
-    TaskType,
-)
 from mteb.benchmarks.benchmark import Benchmark
 from mteb.models import ModelMeta
 from mteb.models.get_model_meta import get_model_metas
-from mteb.types import (
-    ISOLanguage,
-    ISOLanguageScript,
-    Modalities,
-    Score,
-    ScoresDict,
-    SplitName,
-)
 
 from .model_result import ModelResult, _aggregate_and_pivot
 
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator
+
+    from typing_extensions import Self
+
+    from mteb.abstasks.abstask import AbsTask
+    from mteb.abstasks.task_metadata import (
+        TaskDomain,
+        TaskType,
+    )
+    from mteb.types import (
+        ISOLanguage,
+        ISOLanguageScript,
+        Modalities,
+        Score,
+        ScoresDict,
+        SplitName,
+    )
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -144,7 +149,7 @@ class BenchmarkResults(BaseModel):
                     raise ValueError("name in ModelMeta is None. It must be a string.")
                 name_rev[name.name] = name.revision
             else:
-                name_ = cast(str, name)
+                name_ = cast("str", name)
                 name_rev[name_] = revision
 
         for model_res in self.model_results:

mteb/results/model_result.py

@@ -2,30 +2,36 @@ from __future__ import annotations
 
 import logging
 import warnings
-from collections.abc import Callable, Iterable
-from typing import Any, Literal, cast
+from typing import TYPE_CHECKING, Any, Literal, cast
 
 import numpy as np
 import pandas as pd
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import overload
 
-from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.task_metadata import (
-    TaskDomain,
-    TaskType,
-)
 from mteb.types import (
-    ISOLanguage,
-    ISOLanguageScript,
     Modalities,
-    Score,
-    ScoresDict,
-    SplitName,
 )
 
 from .task_result import TaskError, TaskResult
 
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
+    from mteb.abstasks.abstask import AbsTask
+    from mteb.abstasks.task_metadata import (
+        TaskDomain,
+        TaskType,
+    )
+    from mteb.types import (
+        ISOLanguage,
+        ISOLanguageScript,
+        Score,
+        ScoresDict,
+        SplitName,
+    )
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -83,7 +89,7 @@ class ModelResult(BaseModel):
     model_revision: str | None
     task_results: list[TaskResult]
     default_modalities: list[Modalities] = Field(
-        default_factory=lambda: [cast(Modalities, "text")], alias="modalities"
+        default_factory=lambda: [cast("Modalities", "text")], alias="modalities"
     )
     model_config = (
         ConfigDict(  # to free up the name model_* which is otherwise protected
@@ -202,8 +208,8 @@ class ModelResult(BaseModel):
             aggregation = aggregation if aggregation is not None else np.mean
         else:
             use_fast = True
-        aggregation = cast(Callable[[list[Score]], Any], aggregation)
-        getter = cast(Callable[[ScoresDict], Score], getter)
+        aggregation = cast("Callable[[list[Score]], Any]", aggregation)
+        getter = cast("Callable[[ScoresDict], Score]", getter)
 
         if format == "wide":
             scores = {}

mteb/results/task_result.py

@@ -4,34 +4,40 @@ import json
 import logging
 import warnings
 from collections import defaultdict
-from collections.abc import Callable, Iterable, Mapping
 from functools import cached_property
 from importlib.metadata import version
-from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from huggingface_hub import EvalResult
 from packaging.version import Version
 from pydantic import BaseModel, field_validator
-from typing_extensions import Self
 
 from mteb import TaskMetadata
 from mteb._helpful_enum import HelpfulStrEnum
 from mteb.abstasks import AbsTaskClassification
 from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.task_metadata import TaskDomain
 from mteb.languages import LanguageScripts
 from mteb.models.model_meta import ScoringFunction
 from mteb.types import (
-    HFSubset,
-    ISOLanguage,
-    ISOLanguageScript,
-    Score,
     ScoresDict,
     SplitName,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Mapping
+    from pathlib import Path
+
+    from typing_extensions import Self
+
+    from mteb.abstasks.task_metadata import TaskDomain
+    from mteb.types import (
+        HFSubset,
+        ISOLanguage,
+        ISOLanguageScript,
+        Score,
+    )
+
 logger = logging.getLogger(__name__)
 
 

mteb/similarity_functions.py

@@ -1,8 +1,14 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 import torch
 
-from mteb.models import EncoderProtocol
 from mteb.models.model_meta import ScoringFunction
-from mteb.types import Array
+
+if TYPE_CHECKING:
+    from mteb.models import EncoderProtocol
+    from mteb.types import Array
 
 
 def _use_torch_compile():

mteb/tasks/aggregated_tasks/eng/cqadupstack_retrieval.py

@@ -1,5 +1,5 @@
-from mteb.abstasks import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.retrieval import (
     CQADupstackAndroidRetrieval,
     CQADupstackEnglishRetrieval,
@@ -15,7 +15,7 @@ from mteb.tasks.retrieval import (
     CQADupstackWordpressRetrieval,
 )
 
-task_list_cqa: list[AbsTask] = [
+task_list_cqa = [
     CQADupstackAndroidRetrieval(),
     CQADupstackEnglishRetrieval(),
     CQADupstackGamingRetrieval(),

mteb/tasks/aggregated_tasks/eng/sts17_multilingual_visual_sts_eng.py

@@ -1,10 +1,10 @@
-from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.sts.multilingual.sts17_multilingual_visual_sts import (
     STS17MultilingualVisualSTS,
 )
 
-task_list_sts17: list[AbsTask] = [
+task_list_sts17 = [
     STS17MultilingualVisualSTS().filter_languages(
         languages=["eng"], hf_subsets=["en-en"]
     )

mteb/tasks/aggregated_tasks/eng/sts_benchmark_multilingual_visual_sts_eng.py

@@ -1,10 +1,10 @@
-from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.sts.multilingual.sts_benchmark_multilingual_visual_sts import (
     STSBenchmarkMultilingualVisualSTS,
 )
 
-task_list_stsb: list[AbsTask] = [
+task_list_stsb = [
     STSBenchmarkMultilingualVisualSTS().filter_languages(
         languages=["eng"], hf_subsets=["en"]
     )

mteb/tasks/aggregated_tasks/fas/cqadupstack_retrieval_fa.py

@@ -1,5 +1,5 @@
-from mteb.abstasks import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.retrieval import (
     CQADupstackAndroidRetrievalFa,
     CQADupstackEnglishRetrievalFa,
@@ -15,7 +15,7 @@ from mteb.tasks.retrieval import (
     CQADupstackWordpressRetrievalFa,
 )
 
-task_list_cqa: list[AbsTask] = [
+task_list_cqa = [
     CQADupstackAndroidRetrievalFa(),
     CQADupstackEnglishRetrievalFa(),
     CQADupstackGamingRetrievalFa(),

mteb/tasks/aggregated_tasks/fas/syn_per_chatbot_conv_sa_classification.py

@@ -1,5 +1,5 @@
-from mteb.abstasks import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.classification import (
     SynPerChatbotConvSAAnger,
     SynPerChatbotConvSAFear,
@@ -12,7 +12,7 @@ from mteb.tasks.classification import (
     SynPerChatbotConvSASurprise,
 )
 
-task_list_cqa: list[AbsTask] = [
+task_list_cqa = [
     SynPerChatbotConvSAAnger(),
     SynPerChatbotConvSASatisfaction(),
     SynPerChatbotConvSAFriendship(),

mteb/tasks/aggregated_tasks/multilingual/sts17_multilingual_vision_sts.py

@@ -1,10 +1,10 @@
-from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.sts.multilingual.sts17_multilingual_visual_sts import (
     STS17MultilingualVisualSTS,
 )
 
-task_list_sts17_multi: list[AbsTask] = [
+task_list_sts17_multi = [
     STS17MultilingualVisualSTS().filter_languages(
         languages=["ara", "eng", "spa", "kor"],
         hf_subsets=[

mteb/tasks/aggregated_tasks/multilingual/sts_benchmark_multilingual_visual_sts.py

@@ -1,10 +1,10 @@
-from mteb.abstasks.abstask import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.sts.multilingual.sts_benchmark_multilingual_visual_sts import (
     STSBenchmarkMultilingualVisualSTS,
 )
 
-task_list_multi: list[AbsTask] = [
+task_list_multi = [
     STSBenchmarkMultilingualVisualSTS().filter_languages(
         languages=[
             "deu",

mteb/tasks/aggregated_tasks/nld/cqadupstack_nl_retrieval.py

@@ -1,5 +1,5 @@
-from mteb.abstasks import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.retrieval import (
     CQADupstackAndroidNLRetrieval,
     CQADupstackEnglishNLRetrieval,
@@ -15,7 +15,7 @@ from mteb.tasks.retrieval import (
     CQADupstackWordpressNLRetrieval,
 )
 
-task_list_cqa: list[AbsTask] = [
+task_list_cqa = [
     CQADupstackAndroidNLRetrieval(),
     CQADupstackEnglishNLRetrieval(),
     CQADupstackGamingNLRetrieval(),

mteb/tasks/aggregated_tasks/pol/cqadupstack_retrieval_pl.py

@@ -1,5 +1,5 @@
-from mteb.abstasks import AbsTask
-from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata
+from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.tasks.retrieval.pol.cqadupstack_pl_retrieval import (
     CQADupstackAndroidRetrievalPL,
     CQADupstackEnglishRetrievalPL,
@@ -15,7 +15,7 @@ from mteb.tasks.retrieval.pol.cqadupstack_pl_retrieval import (
     CQADupstackWordpressRetrievalPL,
 )
 
-task_list_cqa: list[AbsTask] = [
+task_list_cqa = [
     CQADupstackAndroidRetrievalPL(),
     CQADupstackEnglishRetrievalPL(),
     CQADupstackGamingRetrievalPL(),

mteb/tasks/clustering/nob/snl_clustering.py

@@ -1,13 +1,18 @@
+from __future__ import annotations
+
 import random
-from collections.abc import Iterable
 from itertools import islice
-from typing import TypeVar
+from typing import TYPE_CHECKING, TypeVar
 
 import datasets
 
 from mteb.abstasks.clustering_legacy import AbsTaskClusteringLegacy
 from mteb.abstasks.task_metadata import TaskMetadata
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
 T = TypeVar("T")
 
 

mteb/tasks/clustering/nob/vg_clustering.py

@@ -1,13 +1,18 @@
+from __future__ import annotations
+
 import random
-from collections.abc import Iterable
 from itertools import islice
-from typing import TypeVar
+from typing import TYPE_CHECKING, TypeVar
 
 import datasets
 
 from mteb.abstasks.clustering_legacy import AbsTaskClusteringLegacy
 from mteb.abstasks.task_metadata import TaskMetadata
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
 T = TypeVar("T")
 
 

mteb/tasks/retrieval/eng/__init__.py

@@ -14,6 +14,28 @@ from .birco_whats_that_book_reranking import BIRCOWhatsThatBookReranking
 from .blink_it2i_retrieval import BLINKIT2IRetrieval
 from .blink_it2t_retrieval import BLINKIT2TRetrieval
 from .bright_retrieval import BrightLongRetrieval, BrightRetrieval
+from .bright_v1_1_retrieval import (
+    BrightAopsRetrieval,
+    BrightBiologyLongRetrieval,
+    BrightBiologyRetrieval,
+    BrightEarthScienceLongRetrieval,
+    BrightEarthScienceRetrieval,
+    BrightEconomicsLongRetrieval,
+    BrightEconomicsRetrieval,
+    BrightLeetcodeRetrieval,
+    BrightPonyLongRetrieval,
+    BrightPonyRetrieval,
+    BrightPsychologyLongRetrieval,
+    BrightPsychologyRetrieval,
+    BrightRoboticsLongRetrieval,
+    BrightRoboticsRetrieval,
+    BrightStackoverflowLongRetrieval,
+    BrightStackoverflowRetrieval,
+    BrightSustainableLivingLongRetrieval,
+    BrightSustainableLivingRetrieval,
+    BrightTheoremQAQuestionsRetrieval,
+    BrightTheoremQATheoremsRetrieval,
+)
 from .built_bench_retrieval import BuiltBenchRetrieval
 from .chat_doctor_retrieval import ChatDoctorRetrieval
 from .chem_hotpot_qa_retrieval import ChemHotpotQARetrieval
@@ -236,8 +258,28 @@ __all__ = [
     "BarExamQARetrieval",
     "BillSumCARetrieval",
     "BillSumUSRetrieval",
+    "BrightAopsRetrieval",
+    "BrightBiologyLongRetrieval",
+    "BrightBiologyRetrieval",
+    "BrightEarthScienceLongRetrieval",
+    "BrightEarthScienceRetrieval",
+    "BrightEconomicsLongRetrieval",
+    "BrightEconomicsRetrieval",
+    "BrightLeetcodeRetrieval",
     "BrightLongRetrieval",
+    "BrightPonyLongRetrieval",
+    "BrightPonyRetrieval",
+    "BrightPsychologyLongRetrieval",
+    "BrightPsychologyRetrieval",
     "BrightRetrieval",
+    "BrightRoboticsLongRetrieval",
+    "BrightRoboticsRetrieval",
+    "BrightStackoverflowLongRetrieval",
+    "BrightStackoverflowRetrieval",
+    "BrightSustainableLivingLongRetrieval",
+    "BrightSustainableLivingRetrieval",
+    "BrightTheoremQAQuestionsRetrieval",
+    "BrightTheoremQATheoremsRetrieval",
     "BuiltBenchRetrieval",
     "CIRRIT2IRetrieval",
     "CQADupstackAndroidRetrieval",

mteb/tasks/retrieval/eng/bright_retrieval.py

@@ -1,3 +1,4 @@
+import warnings
 from collections import defaultdict
 
 import datasets
@@ -86,6 +87,12 @@ def load_data(self) -> None:
     if self.data_loaded:
         return
 
+    warnings.warn(
+        "This task contains wrong prompts in the metadata. "
+        "Please use BRIGHT(v1.1) benchmark instead.",
+        category=DeprecationWarning,
+    )
+
     self.corpus, self.queries, self.relevant_docs = self.load_bright_data(
         path=self.metadata.dataset["path"],
         domains=list(self.metadata.eval_langs.keys()),
@@ -104,7 +111,7 @@ class BrightRetrieval(AbsTaskRetrieval):
             "revision": "a75a0eb483f6a5233a6efc2d63d71540a4443dfb",
         },
         reference="https://huggingface.co/datasets/xlangai/BRIGHT",
-        description="Bright retrieval dataset.",
+        description="BRIGHT: A Realistic and Challenging Benchmark for Reasoning-Intensive Retrieval",
         type="Retrieval",
         category="t2t",
         eval_splits=["standard"],
@@ -129,6 +136,7 @@ class BrightRetrieval(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+        superseded_by="BrightBiologyRetrieval",
     )
     load_bright_data = load_bright_data
     load_data = load_data