mteb 2.7.20__py3-none-any.whl → 2.7.22__py3-none-any.whl

mteb/tasks/retrieval/fas/beir_fa.py

@@ -1,6 +1,13 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
+fa_mteb = """@article{zinvandi2025famteb,
+  author = {Zinvandi, Erfan and Alikhani, Morteza and Sarmadi, Mehran and Pourbahman, Zahra and Arvin, Sepehr and Kazemi, Reza and Amini, Arash},
+  journal = {arXiv preprint arXiv:2502.11571},
+  title = {Famteb: Massive text embedding benchmark in persian language},
+  year = {2025},
+}"""
+
 
 class ArguAnaFa(AbsTaskRetrieval):
     ignore_identical_ids = True
@@ -25,7 +32,7 @@ class ArguAnaFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["ArguAna"],
         superseded_by="ArguAna-Fa.v2",
     )
@@ -54,7 +61,7 @@ class ArguAnaFaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["ArguAna"],
     )
 
@@ -81,7 +88,7 @@ class ClimateFEVERFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["ClimateFEVER"],
     )
 
@@ -108,7 +115,7 @@ class CQADupstackAndroidRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackAndroidRetrieval"],
     )
 
@@ -135,7 +142,7 @@ class CQADupstackEnglishRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackEnglishRetrieval"],
     )
 
@@ -162,7 +169,7 @@ class CQADupstackGamingRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackGamingRetrieval"],
     )
 
@@ -189,7 +196,7 @@ class CQADupstackGisRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackGisRetrieval"],
     )
 
@@ -216,7 +223,7 @@ class CQADupstackMathematicaRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackMathematicaRetrieval"],
     )
 
@@ -243,7 +250,7 @@ class CQADupstackPhysicsRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackPhysicsRetrieval"],
     )
 
@@ -270,7 +277,7 @@ class CQADupstackProgrammersRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackProgrammersRetrieval"],
     )
 
@@ -297,7 +304,7 @@ class CQADupstackStatsRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackStatsRetrieval"],
     )
 
@@ -324,7 +331,7 @@ class CQADupstackTexRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackTexRetrieval"],
     )
 
@@ -351,7 +358,7 @@ class CQADupstackUnixRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackUnixRetrieval"],
     )
 
@@ -378,7 +385,7 @@ class CQADupstackWebmastersRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackWebmastersRetrieval"],
     )
 
@@ -405,7 +412,7 @@ class CQADupstackWordpressRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["CQADupstackWordpressRetrieval"],
     )
 
@@ -432,7 +439,7 @@ class DBPediaFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["DBPedia"],
     )
 
@@ -460,7 +467,7 @@ class FiQA2018Fa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["FiQA2018"],
         superseded_by="FiQA2018-Fa.v2",
     )
@@ -490,7 +497,7 @@ class FiQA2018FaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["FiQA2018"],
     )
 
@@ -517,7 +524,7 @@ class HotpotQAFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["HotpotQA"],
     )
 
@@ -546,7 +553,7 @@ class MSMARCOFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["MSMARCO"],
     )
 
@@ -573,7 +580,7 @@ class NFCorpusFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["NFCorpus"],
     )
 
@@ -600,7 +607,7 @@ class NQFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["NQ"],
     )
 
@@ -629,7 +636,7 @@ class QuoraRetrievalFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["QuoraRetrieval"],
         superseded_by="QuoraRetrieval-Fa.v2",
     )
@@ -659,7 +666,7 @@ class QuoraRetrievalFaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["QuoraRetrieval"],
     )
 
@@ -686,7 +693,7 @@ class SCIDOCSFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["SCIDOCS"],
         superseded_by="SCIDOCS-Fa.v2",
     )
@@ -714,7 +721,7 @@ class SCIDOCSFaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["SCIDOCS"],
     )
 
@@ -741,7 +748,7 @@ class SciFactFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["SciFact"],
         superseded_by="SciFact-Fa.v2",
     )
@@ -769,7 +776,7 @@ class SciFactFaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["SciFact"],
     )
 
@@ -796,7 +803,7 @@ class TRECCOVIDFa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["TRECCOVID"],
         superseded_by="TRECCOVID-Fa.v2",
     )
@@ -824,7 +831,7 @@ class TRECCOVIDFaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["TRECCOVID"],
     )
 
@@ -851,7 +858,7 @@ class Touche2020Fa(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["Touche2020"],
         superseded_by="Touche2020-Fa.v2",
     )
@@ -879,7 +886,7 @@ class Touche2020FaV2(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["Touche2020"],
     )
 
@@ -906,7 +913,7 @@ class HotpotQAFaHardNegatives(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["HotpotQA"],
     )
 
@@ -935,7 +942,7 @@ class MSMARCOFaHardNegatives(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["MSMARCO"],
     )
 
@@ -962,7 +969,7 @@ class NQFaHardNegatives(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["NQ"],
     )
 
@@ -991,6 +998,6 @@ class FEVERFaHardNegatives(AbsTaskRetrieval):
         annotations_creators="human-annotated",
         dialect=[],
         sample_creation="found",
-        bibtex_citation=""" """,
+        bibtex_citation=fa_mteb,
         adapted_from=["FEVER"],
     )

mteb/tasks/retrieval/fas/fa_mteb_retrieval.py

@@ -1,6 +1,13 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
+fa_mteb = """@article{zinvandi2025famteb,
+  author = {Zinvandi, Erfan and Alikhani, Morteza and Sarmadi, Mehran and Pourbahman, Zahra and Arvin, Sepehr and Kazemi, Reza and Amini, Arash},
+  journal = {arXiv preprint arXiv:2502.11571},
+  title = {Famteb: Massive text embedding benchmark in persian language},
+  year = {2025},
+}"""
+
 
 class SynPerQARetrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
@@ -25,7 +32,7 @@ class SynPerQARetrieval(AbsTaskRetrieval):
         annotations_creators="LM-generated",
         dialect=[],
         sample_creation="LM-generated and verified",
-        bibtex_citation="""""",
+        bibtex_citation=fa_mteb,
     )
 
 
@@ -52,7 +59,7 @@ class SynPerChatbotTopicsRetrieval(AbsTaskRetrieval):
         annotations_creators="LM-generated",
         dialect=[],
         sample_creation="LM-generated and verified",
-        bibtex_citation="""""",
+        bibtex_citation=fa_mteb,
     )
 
 
@@ -79,7 +86,7 @@ class SynPerChatbotRAGTopicsRetrieval(AbsTaskRetrieval):
         annotations_creators="LM-generated",
         dialect=[],
         sample_creation="LM-generated and verified",
-        bibtex_citation="""""",
+        bibtex_citation=fa_mteb,
     )
 
 
@@ -106,7 +113,7 @@ class SynPerChatbotRAGFAQRetrieval(AbsTaskRetrieval):
         annotations_creators="LM-generated",
         dialect=[],
         sample_creation="LM-generated and verified",
-        bibtex_citation="""""",
+        bibtex_citation=fa_mteb,
     )
 
 
@@ -133,5 +140,5 @@ class PersianWebDocumentRetrieval(AbsTaskRetrieval):
         annotations_creators="derived",
         dialect=[],
         sample_creation="found",
-        bibtex_citation="""""",
+        bibtex_citation=fa_mteb,
     )

mteb/tasks/retrieval/fra/alloprof_retrieval.py

@@ -17,7 +17,7 @@ class AlloprofRetrieval(AbsTaskRetrieval):
         eval_splits=["test"],
         eval_langs=["fra-Latn"],
         main_score="ndcg_at_10",
-        date=None,  # no date specified.
+        date=("2023-02-01", "2023-02-28"),  # publication year
         domains=["Encyclopaedic", "Written"],
         task_subtypes=[],
         license="cc-by-nc-sa-4.0",

mteb/tasks/retrieval/fra/syntec_retrieval.py

@@ -19,7 +19,7 @@ class SyntecRetrieval(AbsTaskRetrieval):
         eval_splits=["test"],
         eval_langs=["fra-Latn"],
         main_score="ndcg_at_10",
-        date=None,  # not specified
+        date=("2024-01-01", "2024-12-31"),  # publication year
         domains=["Legal", "Written"],
         task_subtypes=[],
         license="not specified",

mteb/tasks/retrieval/pol/fi_qapl_retrieval.py

@@ -19,7 +19,7 @@ class FiQAPLRetrieval(AbsTaskRetrieval):
         eval_splits=["test"],
         eval_langs=["pol-Latn"],
         main_score="ndcg_at_10",
-        date=None,
+        date=("2018-01-01", "2018-12-31"),  # publication year
         domains=["Written", "Financial"],
         task_subtypes=["Question answering"],
         license="not specified",

mteb/tasks/retrieval/pol/nqpl_retrieval.py

@@ -53,11 +53,11 @@ class NQPLHardNegatives(AbsTaskRetrieval):
         eval_splits=["test"],
         eval_langs=["pol-Latn"],
         main_score="ndcg_at_10",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
+        date=("2019-01-01", "2019-12-31"),  # original publication year
+        domains=["Written", "Encyclopaedic"],
+        task_subtypes=["Question answering"],
+        license="cc-by-nc-sa-3.0",  # original license
+        annotations_creators="human-annotated",  # original annotation method
         dialect=[],
         sample_creation="machine-translated",
         bibtex_citation=r"""

mteb/tasks/sts/eng/biosses_sts.py

@@ -17,7 +17,7 @@ class BiossesSTS(AbsTaskSTS):
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="cosine_spearman",
-        date=None,
+        date=("2017-07-01", "2017-12-31"),  # publication date
         domains=["Medical"],
         task_subtypes=[],
         license="not specified",

mteb/tasks/sts/eng/humests_benchmark.py

@@ -17,7 +17,7 @@ class HUMESTSBenchmark(AbsTaskSTS):
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="cosine_spearman",
-        date=None,
+        date=("2021-01-01", "2021-12-31"),  # publication year
         domains=["Blog", "News", "Written"],
         task_subtypes=[],
         license="not specified",

mteb/tasks/sts/eng/sts_benchmark_sts.py

@@ -20,7 +20,7 @@ class STSBenchmarkSTS(AbsTaskSTS):
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="cosine_spearman",
-        date=None,
+        date=("2021-01-01", "2021-12-31"),  # publication year
         domains=["Blog", "News", "Written"],
         task_subtypes=[],
         license="not specified",

mteb/tasks/sts/fin/fin_para_sts.py

@@ -21,7 +21,7 @@ class FinParaSTS(AbsTaskSTS):
         main_score="cosine_spearman",
         date=("2017-01-01", "2021-12-31"),
         domains=["News", "Subtitles", "Written"],
-        task_subtypes=None,
+        task_subtypes=[],
         license="cc-by-sa-4.0",
         annotations_creators="expert-annotated",
         dialect=[],

mteb/tasks/sts/kor/klue_sts.py

@@ -20,7 +20,7 @@ class KlueSTS(AbsTaskSTS):
         main_score="cosine_spearman",
         date=("2011-01-01", "2021-11-02"),  # rough estimate,
         domains=["Reviews", "News", "Spoken", "Written", "Spoken"],
-        task_subtypes=None,
+        task_subtypes=[],
         license="cc-by-sa-4.0",
         annotations_creators="human-annotated",
         dialect=[],

mteb/tasks/sts/ron/ron_sts.py

@@ -19,7 +19,7 @@ class RonSTS(AbsTaskSTS):
         main_score="cosine_spearman",
         date=("2020-01-01", "2021-01-31"),
         domains=["News", "Social", "Web", "Written"],  # web for image captions
-        task_subtypes=None,
+        task_subtypes=[],
         license="cc-by-4.0",  # not specified
         annotations_creators="human-annotated",
         dialect=[],

{mteb-2.7.20.dist-info → mteb-2.7.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.20
+Version: 2.7.22
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>
@@ -104,6 +104,16 @@ Requires-Dist: tencentcloud-sdk-python-common>=3.0.1454; extra == "youtu"
 Requires-Dist: tencentcloud-sdk-python-lkeap>=3.0.1451; extra == "youtu"
 Provides-Extra: llama-embed-nemotron
 Requires-Dist: transformers==4.51.0; extra == "llama-embed-nemotron"
+Provides-Extra: llama-nemotron-colembed-vl
+Requires-Dist: transformers[torch]==4.49.0; extra == "llama-nemotron-colembed-vl"
+Requires-Dist: torchvision>=0.22.0; extra == "llama-nemotron-colembed-vl"
+Requires-Dist: flash-attn>=2.6.3; extra == "llama-nemotron-colembed-vl"
+Requires-Dist: accelerate; extra == "llama-nemotron-colembed-vl"
+Provides-Extra: nemotron-colembed-vl-v2
+Requires-Dist: transformers[torch]==5.0.0rc0; extra == "nemotron-colembed-vl-v2"
+Requires-Dist: torchvision>=0.22.0; extra == "nemotron-colembed-vl-v2"
+Requires-Dist: flash-attn>=2.6.3; extra == "nemotron-colembed-vl-v2"
+Requires-Dist: accelerate; extra == "nemotron-colembed-vl-v2"
 Provides-Extra: faiss-cpu
 Requires-Dist: faiss-cpu>=1.12.0; extra == "faiss-cpu"
 Provides-Extra: eager-embed