mteb 2.5.2__py3-none-any.whl → 2.7.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mteb/__init__.py +2 -0
- mteb/_create_dataloaders.py +17 -18
- mteb/_evaluators/any_sts_evaluator.py +3 -3
- mteb/_evaluators/clustering_evaluator.py +2 -2
- mteb/_evaluators/evaluator.py +4 -2
- mteb/_evaluators/image/imagetext_pairclassification_evaluator.py +10 -8
- mteb/_evaluators/pair_classification_evaluator.py +5 -3
- mteb/_evaluators/retrieval_evaluator.py +2 -2
- mteb/_evaluators/retrieval_metrics.py +18 -17
- mteb/_evaluators/sklearn_evaluator.py +11 -10
- mteb/_evaluators/text/bitext_mining_evaluator.py +27 -18
- mteb/_evaluators/text/summarization_evaluator.py +23 -18
- mteb/_evaluators/zeroshot_classification_evaluator.py +5 -3
- mteb/abstasks/_data_filter/filters.py +1 -1
- mteb/abstasks/_data_filter/task_pipelines.py +3 -0
- mteb/abstasks/_statistics_calculation.py +18 -10
- mteb/abstasks/_stratification.py +18 -18
- mteb/abstasks/abstask.py +35 -28
- mteb/abstasks/aggregate_task_metadata.py +1 -9
- mteb/abstasks/aggregated_task.py +10 -29
- mteb/abstasks/classification.py +15 -10
- mteb/abstasks/clustering.py +19 -15
- mteb/abstasks/clustering_legacy.py +10 -10
- mteb/abstasks/image/image_text_pair_classification.py +7 -4
- mteb/abstasks/multilabel_classification.py +23 -19
- mteb/abstasks/pair_classification.py +20 -11
- mteb/abstasks/regression.py +4 -4
- mteb/abstasks/retrieval.py +28 -24
- mteb/abstasks/retrieval_dataset_loaders.py +2 -2
- mteb/abstasks/sts.py +8 -5
- mteb/abstasks/task_metadata.py +31 -33
- mteb/abstasks/text/bitext_mining.py +39 -28
- mteb/abstasks/text/reranking.py +8 -6
- mteb/abstasks/text/summarization.py +10 -5
- mteb/abstasks/zeroshot_classification.py +8 -4
- mteb/benchmarks/benchmark.py +4 -2
- mteb/benchmarks/benchmarks/__init__.py +4 -0
- mteb/benchmarks/benchmarks/benchmarks.py +112 -11
- mteb/benchmarks/get_benchmark.py +14 -55
- mteb/cache.py +182 -29
- mteb/cli/_display_tasks.py +2 -2
- mteb/cli/build_cli.py +110 -14
- mteb/cli/generate_model_card.py +43 -23
- mteb/deprecated_evaluator.py +63 -49
- mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2CybersecurityRetrieval.json +32 -0
- mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EconomicRetrieval.json +32 -0
- mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EnergyRetrieval.json +32 -0
- mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2HrRetrieval.json +32 -0
- mteb/descriptive_stats/Retrieval/ChemRxivRetrieval.json +30 -0
- mteb/descriptive_stats/Retrieval/EuroPIRQRetrieval.json +116 -0
- mteb/descriptive_stats/Retrieval/NanoClimateFEVER-VN.json +30 -0
- mteb/descriptive_stats/Retrieval/NanoDBPedia-VN.json +30 -0
- mteb/descriptive_stats/Retrieval/NanoFEVER-VN.json +30 -0
- mteb/descriptive_stats/Retrieval/NanoHotpotQA-VN.json +30 -0
- mteb/descriptive_stats/Retrieval/NanoMSMARCO-VN.json +30 -0
- mteb/descriptive_stats/Retrieval/NanoNQ-VN.json +30 -0
- mteb/descriptive_stats/Retrieval/TVPLRetrieval.json +30 -0
- mteb/evaluate.py +44 -33
- mteb/filter_tasks.py +25 -26
- mteb/get_tasks.py +29 -30
- mteb/languages/language_scripts.py +5 -3
- mteb/leaderboard/app.py +162 -34
- mteb/load_results.py +12 -12
- mteb/models/abs_encoder.py +10 -6
- mteb/models/cache_wrappers/cache_backend_protocol.py +3 -5
- mteb/models/cache_wrappers/cache_backends/_hash_utils.py +5 -4
- mteb/models/cache_wrappers/cache_backends/faiss_cache.py +6 -2
- mteb/models/cache_wrappers/cache_backends/numpy_cache.py +43 -25
- mteb/models/cache_wrappers/cache_wrapper.py +2 -2
- mteb/models/get_model_meta.py +21 -3
- mteb/models/instruct_wrapper.py +28 -8
- mteb/models/model_implementations/align_models.py +1 -1
- mteb/models/model_implementations/andersborges.py +4 -4
- mteb/models/model_implementations/ara_models.py +1 -1
- mteb/models/model_implementations/arctic_models.py +8 -8
- mteb/models/model_implementations/b1ade_models.py +1 -1
- mteb/models/model_implementations/bge_models.py +45 -21
- mteb/models/model_implementations/bica_model.py +3 -3
- mteb/models/model_implementations/blip2_models.py +2 -2
- mteb/models/model_implementations/blip_models.py +16 -16
- mteb/models/model_implementations/bm25.py +4 -4
- mteb/models/model_implementations/bmretriever_models.py +6 -4
- mteb/models/model_implementations/cadet_models.py +1 -1
- mteb/models/model_implementations/cde_models.py +11 -4
- mteb/models/model_implementations/clip_models.py +6 -6
- mteb/models/model_implementations/clips_models.py +3 -3
- mteb/models/model_implementations/codefuse_models.py +5 -5
- mteb/models/model_implementations/codesage_models.py +3 -3
- mteb/models/model_implementations/cohere_models.py +5 -5
- mteb/models/model_implementations/cohere_v.py +2 -2
- mteb/models/model_implementations/colpali_models.py +3 -3
- mteb/models/model_implementations/colqwen_models.py +8 -8
- mteb/models/model_implementations/colsmol_models.py +2 -2
- mteb/models/model_implementations/conan_models.py +1 -1
- mteb/models/model_implementations/dino_models.py +42 -42
- mteb/models/model_implementations/e5_instruct.py +23 -4
- mteb/models/model_implementations/e5_models.py +9 -9
- mteb/models/model_implementations/e5_v.py +6 -6
- mteb/models/model_implementations/eagerworks_models.py +1 -1
- mteb/models/model_implementations/emillykkejensen_models.py +6 -6
- mteb/models/model_implementations/en_code_retriever.py +1 -1
- mteb/models/model_implementations/euler_models.py +2 -2
- mteb/models/model_implementations/fa_models.py +9 -9
- mteb/models/model_implementations/facebookai.py +14 -2
- mteb/models/model_implementations/geogpt_models.py +1 -1
- mteb/models/model_implementations/gme_v_models.py +6 -5
- mteb/models/model_implementations/google_models.py +1 -1
- mteb/models/model_implementations/granite_vision_embedding_models.py +1 -1
- mteb/models/model_implementations/gritlm_models.py +2 -2
- mteb/models/model_implementations/gte_models.py +25 -13
- mteb/models/model_implementations/hinvec_models.py +1 -1
- mteb/models/model_implementations/ibm_granite_models.py +30 -6
- mteb/models/model_implementations/inf_models.py +2 -2
- mteb/models/model_implementations/jasper_models.py +2 -2
- mteb/models/model_implementations/jina_clip.py +48 -10
- mteb/models/model_implementations/jina_models.py +18 -11
- mteb/models/model_implementations/kblab.py +12 -6
- mteb/models/model_implementations/kennethenevoldsen_models.py +4 -4
- mteb/models/model_implementations/kfst.py +1 -1
- mteb/models/model_implementations/kowshik24_models.py +1 -1
- mteb/models/model_implementations/lgai_embedding_models.py +1 -1
- mteb/models/model_implementations/linq_models.py +1 -1
- mteb/models/model_implementations/listconranker.py +1 -1
- mteb/models/model_implementations/llm2clip_models.py +6 -6
- mteb/models/model_implementations/llm2vec_models.py +8 -8
- mteb/models/model_implementations/mcinext_models.py +4 -1
- mteb/models/model_implementations/mdbr_models.py +17 -3
- mteb/models/model_implementations/misc_models.py +68 -68
- mteb/models/model_implementations/mixedbread_ai_models.py +332 -0
- mteb/models/model_implementations/mme5_models.py +1 -1
- mteb/models/model_implementations/moco_models.py +4 -4
- mteb/models/model_implementations/mod_models.py +1 -1
- mteb/models/model_implementations/model2vec_models.py +14 -14
- mteb/models/model_implementations/moka_models.py +1 -1
- mteb/models/model_implementations/nbailab.py +3 -3
- mteb/models/model_implementations/no_instruct_sentence_models.py +2 -2
- mteb/models/model_implementations/nomic_models.py +30 -15
- mteb/models/model_implementations/nomic_models_vision.py +1 -1
- mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py +15 -9
- mteb/models/model_implementations/nvidia_models.py +151 -19
- mteb/models/model_implementations/octen_models.py +61 -2
- mteb/models/model_implementations/openclip_models.py +13 -13
- mteb/models/model_implementations/opensearch_neural_sparse_models.py +5 -5
- mteb/models/model_implementations/ops_moa_models.py +1 -1
- mteb/models/model_implementations/ordalietech_solon_embeddings_mini_beta_1_1.py +1 -1
- mteb/models/model_implementations/pawan_models.py +1 -1
- mteb/models/model_implementations/piccolo_models.py +1 -1
- mteb/models/model_implementations/pixie_models.py +56 -0
- mteb/models/model_implementations/promptriever_models.py +4 -4
- mteb/models/model_implementations/pylate_models.py +10 -9
- mteb/models/model_implementations/qodo_models.py +2 -2
- mteb/models/model_implementations/qtack_models.py +1 -1
- mteb/models/model_implementations/qwen3_models.py +3 -3
- mteb/models/model_implementations/qzhou_models.py +2 -2
- mteb/models/model_implementations/random_baseline.py +3 -3
- mteb/models/model_implementations/rasgaard_models.py +2 -2
- mteb/models/model_implementations/reasonir_model.py +1 -1
- mteb/models/model_implementations/repllama_models.py +3 -3
- mteb/models/model_implementations/rerankers_custom.py +12 -6
- mteb/models/model_implementations/rerankers_monot5_based.py +17 -17
- mteb/models/model_implementations/richinfoai_models.py +1 -1
- mteb/models/model_implementations/ru_sentence_models.py +20 -20
- mteb/models/model_implementations/ruri_models.py +10 -10
- mteb/models/model_implementations/salesforce_models.py +3 -3
- mteb/models/model_implementations/samilpwc_models.py +1 -1
- mteb/models/model_implementations/sarashina_embedding_models.py +2 -2
- mteb/models/model_implementations/searchmap_models.py +1 -1
- mteb/models/model_implementations/seed_1_6_embedding_models_1215.py +113 -146
- mteb/models/model_implementations/sentence_transformers_models.py +124 -22
- mteb/models/model_implementations/shuu_model.py +1 -1
- mteb/models/model_implementations/siglip_models.py +20 -20
- mteb/models/model_implementations/slm_models.py +416 -0
- mteb/models/model_implementations/spartan8806_atles_champion.py +1 -1
- mteb/models/model_implementations/stella_models.py +17 -4
- mteb/models/model_implementations/tarka_models.py +2 -2
- mteb/models/model_implementations/text2vec_models.py +9 -3
- mteb/models/model_implementations/ua_sentence_models.py +1 -1
- mteb/models/model_implementations/uae_models.py +7 -1
- mteb/models/model_implementations/vdr_models.py +1 -1
- mteb/models/model_implementations/vi_vn_models.py +6 -6
- mteb/models/model_implementations/vlm2vec_models.py +3 -3
- mteb/models/model_implementations/voyage_models.py +84 -0
- mteb/models/model_implementations/voyage_v.py +9 -7
- mteb/models/model_implementations/youtu_models.py +1 -1
- mteb/models/model_implementations/yuan_models.py +1 -1
- mteb/models/model_implementations/yuan_models_en.py +1 -1
- mteb/models/model_meta.py +80 -31
- mteb/models/models_protocols.py +22 -6
- mteb/models/search_encoder_index/search_indexes/faiss_search_index.py +9 -6
- mteb/models/search_wrappers.py +33 -18
- mteb/models/sentence_transformer_wrapper.py +50 -25
- mteb/models/vllm_wrapper.py +327 -0
- mteb/py.typed +0 -0
- mteb/results/benchmark_results.py +29 -21
- mteb/results/model_result.py +52 -22
- mteb/results/task_result.py +80 -58
- mteb/similarity_functions.py +11 -7
- mteb/tasks/classification/dan/dk_hate_classification.py +1 -1
- mteb/tasks/classification/est/estonian_valence.py +1 -1
- mteb/tasks/classification/kur/kurdish_sentiment_classification.py +2 -2
- mteb/tasks/classification/multilingual/scala_classification.py +1 -1
- mteb/tasks/clustering/eng/hume_wiki_cities_clustering.py +1 -1
- mteb/tasks/clustering/eng/wiki_cities_clustering.py +1 -1
- mteb/tasks/clustering/zho/cmteb_clustering.py +2 -2
- mteb/tasks/image_text_pair_classification/eng/sugar_crepe.py +1 -1
- mteb/tasks/reranking/multilingual/wikipedia_reranking_multilingual.py +1 -1
- mteb/tasks/retrieval/code/code_rag.py +12 -12
- mteb/tasks/retrieval/dan/dan_fever_retrieval.py +1 -1
- mteb/tasks/retrieval/dan/tv2_nordretrieval.py +2 -2
- mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py +2 -2
- mteb/tasks/retrieval/eng/__init__.py +2 -0
- mteb/tasks/retrieval/eng/chemrxiv.py +33 -0
- mteb/tasks/retrieval/eng/cub200_i2i_retrieval.py +1 -1
- mteb/tasks/retrieval/kor/__init__.py +15 -1
- mteb/tasks/retrieval/kor/kovidore2_bench_retrieval.py +142 -0
- mteb/tasks/retrieval/multilingual/__init__.py +2 -0
- mteb/tasks/retrieval/multilingual/euro_pirq_retrieval.py +43 -0
- mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py +90 -100
- mteb/tasks/retrieval/nob/norquad.py +2 -2
- mteb/tasks/retrieval/nob/snl_retrieval.py +2 -2
- mteb/tasks/retrieval/tur/tur_hist_quad.py +1 -1
- mteb/tasks/retrieval/vie/__init__.py +14 -6
- mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py +39 -0
- mteb/tasks/retrieval/vie/db_pedia_vn_retrieval.py +39 -0
- mteb/tasks/retrieval/vie/fevervn_retrieval.py +39 -0
- mteb/tasks/retrieval/vie/hotpot_qavn_retrieval.py +39 -0
- mteb/tasks/retrieval/vie/msmarcovn_retrieval.py +48 -0
- mteb/tasks/retrieval/vie/nqvn_retrieval.py +39 -0
- mteb/tasks/retrieval/vie/tvpl_retrieval.py +42 -0
- mteb/tasks/retrieval/vie/zac_legal_text_retrieval.py +15 -1
- mteb/types/__init__.py +2 -0
- mteb/types/_encoder_io.py +12 -0
- mteb/types/_result.py +2 -1
- mteb/types/statistics.py +9 -3
- {mteb-2.5.2.dist-info → mteb-2.7.2.dist-info}/METADATA +15 -4
- {mteb-2.5.2.dist-info → mteb-2.7.2.dist-info}/RECORD +240 -219
- mteb/models/model_implementations/mxbai_models.py +0 -111
- {mteb-2.5.2.dist-info → mteb-2.7.2.dist-info}/WHEEL +0 -0
- {mteb-2.5.2.dist-info → mteb-2.7.2.dist-info}/entry_points.txt +0 -0
- {mteb-2.5.2.dist-info → mteb-2.7.2.dist-info}/licenses/LICENSE +0 -0
- {mteb-2.5.2.dist-info → mteb-2.7.2.dist-info}/top_level.txt +0 -0
|
@@ -82,7 +82,7 @@ e5_mult_small = ModelMeta(
|
|
|
82
82
|
max_tokens=512,
|
|
83
83
|
reference="https://huggingface.co/intfloat/multilingual-e5-small",
|
|
84
84
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
85
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
85
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
86
86
|
use_instructions=True,
|
|
87
87
|
public_training_code=None, # couldn't find
|
|
88
88
|
public_training_data=None,
|
|
@@ -109,7 +109,7 @@ e5_mult_base = ModelMeta(
|
|
|
109
109
|
max_tokens=514,
|
|
110
110
|
reference="https://huggingface.co/intfloat/multilingual-e5-base",
|
|
111
111
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
112
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
112
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
113
113
|
use_instructions=True,
|
|
114
114
|
public_training_code=None,
|
|
115
115
|
public_training_data=None,
|
|
@@ -136,7 +136,7 @@ e5_mult_large = ModelMeta(
|
|
|
136
136
|
max_tokens=514,
|
|
137
137
|
reference="https://huggingface.co/intfloat/multilingual-e5-large",
|
|
138
138
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
139
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
139
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
140
140
|
use_instructions=True,
|
|
141
141
|
public_training_code=None,
|
|
142
142
|
public_training_data=None,
|
|
@@ -163,7 +163,7 @@ e5_eng_small_v2 = ModelMeta(
|
|
|
163
163
|
max_tokens=512,
|
|
164
164
|
reference="https://huggingface.co/intfloat/e5-small-v2",
|
|
165
165
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
166
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
166
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
167
167
|
use_instructions=True,
|
|
168
168
|
public_training_code=None,
|
|
169
169
|
public_training_data=None,
|
|
@@ -190,7 +190,7 @@ e5_eng_small = ModelMeta(
|
|
|
190
190
|
max_tokens=512,
|
|
191
191
|
reference="https://huggingface.co/intfloat/e5-small",
|
|
192
192
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
193
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
193
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
194
194
|
use_instructions=True,
|
|
195
195
|
public_training_code=None,
|
|
196
196
|
public_training_data=None,
|
|
@@ -217,7 +217,7 @@ e5_eng_base_v2 = ModelMeta(
|
|
|
217
217
|
max_tokens=512,
|
|
218
218
|
reference="https://huggingface.co/intfloat/e5-base-v2",
|
|
219
219
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
220
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
220
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
221
221
|
use_instructions=True,
|
|
222
222
|
superseded_by=None,
|
|
223
223
|
adapted_from="intfloat/e5-base",
|
|
@@ -245,7 +245,7 @@ e5_eng_large_v2 = ModelMeta(
|
|
|
245
245
|
max_tokens=514,
|
|
246
246
|
reference="https://huggingface.co/intfloat/e5-large-v2",
|
|
247
247
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
248
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
248
|
+
framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
|
|
249
249
|
use_instructions=True,
|
|
250
250
|
superseded_by=None,
|
|
251
251
|
adapted_from="intfloat/e5-large",
|
|
@@ -273,7 +273,7 @@ e5_large = ModelMeta(
|
|
|
273
273
|
max_tokens=512,
|
|
274
274
|
reference="https://huggingface.co/intfloat/e5-large",
|
|
275
275
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
276
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
276
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
277
277
|
use_instructions=True,
|
|
278
278
|
superseded_by="intfloat/e5-large-v2",
|
|
279
279
|
adapted_from="google-bert/bert-large-uncased-whole-word-masking",
|
|
@@ -301,7 +301,7 @@ e5_base = ModelMeta(
|
|
|
301
301
|
max_tokens=512,
|
|
302
302
|
reference="https://huggingface.co/intfloat/e5-base",
|
|
303
303
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
304
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
304
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
305
305
|
use_instructions=True,
|
|
306
306
|
superseded_by="intfloat/e5-base-v2",
|
|
307
307
|
adapted_from="google-bert/bert-base-uncased",
|
|
@@ -30,6 +30,7 @@ class E5VModel(AbsEncoder):
|
|
|
30
30
|
self,
|
|
31
31
|
model_name: str,
|
|
32
32
|
revision: str,
|
|
33
|
+
device: str | None = None,
|
|
33
34
|
composed_prompt=None,
|
|
34
35
|
**kwargs: Any,
|
|
35
36
|
):
|
|
@@ -47,8 +48,7 @@ class E5VModel(AbsEncoder):
|
|
|
47
48
|
self.processor = LlavaNextProcessor.from_pretrained(
|
|
48
49
|
model_name, revision=revision
|
|
49
50
|
)
|
|
50
|
-
|
|
51
|
-
self.device = kwargs.pop("device")
|
|
51
|
+
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
52
52
|
self.model = LlavaNextForConditionalGeneration.from_pretrained(
|
|
53
53
|
model_name, revision=revision, **kwargs
|
|
54
54
|
)
|
|
@@ -87,7 +87,7 @@ class E5VModel(AbsEncoder):
|
|
|
87
87
|
],
|
|
88
88
|
return_tensors="pt",
|
|
89
89
|
padding=True,
|
|
90
|
-
).to(
|
|
90
|
+
).to(self.device)
|
|
91
91
|
text_outputs = self.model(
|
|
92
92
|
**text_inputs, output_hidden_states=True, return_dict=True
|
|
93
93
|
).hidden_states[-1][:, -1, :]
|
|
@@ -111,7 +111,7 @@ class E5VModel(AbsEncoder):
|
|
|
111
111
|
batch["image"],
|
|
112
112
|
return_tensors="pt",
|
|
113
113
|
padding=True,
|
|
114
|
-
).to(
|
|
114
|
+
).to(self.device)
|
|
115
115
|
image_outputs = self.model(
|
|
116
116
|
**img_inputs, output_hidden_states=True, return_dict=True
|
|
117
117
|
).hidden_states[-1][:, -1, :]
|
|
@@ -141,7 +141,7 @@ class E5VModel(AbsEncoder):
|
|
|
141
141
|
]
|
|
142
142
|
inputs = self.processor(
|
|
143
143
|
prompts, batch["image"], return_tensors="pt", padding=True
|
|
144
|
-
).to(
|
|
144
|
+
).to(self.device)
|
|
145
145
|
outputs = self.model(
|
|
146
146
|
**inputs, output_hidden_states=True, return_dict=True
|
|
147
147
|
).hidden_states[-1][:, -1, :]
|
|
@@ -173,7 +173,7 @@ e5_v = ModelMeta(
|
|
|
173
173
|
open_weights=True,
|
|
174
174
|
public_training_code="https://github.com/kongds/E5-V",
|
|
175
175
|
public_training_data="https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse",
|
|
176
|
-
framework=["PyTorch"],
|
|
176
|
+
framework=["PyTorch", "Transformers", "safetensors"],
|
|
177
177
|
reference="https://huggingface.co/royokong/e5-v",
|
|
178
178
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
179
179
|
use_instructions=True,
|
|
@@ -152,7 +152,7 @@ Eager_Embed_V1 = ModelMeta(
|
|
|
152
152
|
embed_dim=2560,
|
|
153
153
|
license="apache-2.0",
|
|
154
154
|
open_weights=True,
|
|
155
|
-
framework=["Tevatron"],
|
|
155
|
+
framework=["Tevatron", "safetensors"],
|
|
156
156
|
reference="https://huggingface.co/eagerworks/eager-embed-v1",
|
|
157
157
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
158
158
|
use_instructions=True,
|
|
@@ -2,7 +2,7 @@ from mteb.models.model_meta import ModelMeta
|
|
|
2
2
|
from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
|
|
3
3
|
|
|
4
4
|
embedding_gemma_300m_scandi = ModelMeta(
|
|
5
|
-
loader=sentence_transformers_loader,
|
|
5
|
+
loader=sentence_transformers_loader,
|
|
6
6
|
name="emillykkejensen/EmbeddingGemma-Scandi-300m",
|
|
7
7
|
model_type=["dense"],
|
|
8
8
|
languages=["dan-Latn", "swe-Latn", "nor-Latn", "nob-Latn", "nno-Latn"],
|
|
@@ -14,7 +14,7 @@ embedding_gemma_300m_scandi = ModelMeta(
|
|
|
14
14
|
max_tokens=2048,
|
|
15
15
|
license="apache-2.0",
|
|
16
16
|
reference="https://huggingface.co/emillykkejensen/EmbeddingGemma-Scandi-300m",
|
|
17
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
17
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
18
18
|
use_instructions=True,
|
|
19
19
|
public_training_code=None,
|
|
20
20
|
public_training_data="https://huggingface.co/datasets/DDSC/nordic-embedding-training-data",
|
|
@@ -35,7 +35,7 @@ embedding_gemma_300m_scandi = ModelMeta(
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
qwen_scandi = ModelMeta(
|
|
38
|
-
loader=sentence_transformers_loader,
|
|
38
|
+
loader=sentence_transformers_loader,
|
|
39
39
|
name="emillykkejensen/Qwen3-Embedding-Scandi-0.6B",
|
|
40
40
|
model_type=["dense"],
|
|
41
41
|
languages=["dan-Latn", "swe-Latn", "nor-Latn", "nob-Latn", "nno-Latn"],
|
|
@@ -48,7 +48,7 @@ qwen_scandi = ModelMeta(
|
|
|
48
48
|
max_tokens=32768,
|
|
49
49
|
license="apache-2.0",
|
|
50
50
|
reference="https://huggingface.co/emillykkejensen/Qwen3-Embedding-Scandi-0.6B",
|
|
51
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
51
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
52
52
|
use_instructions=True,
|
|
53
53
|
public_training_code=None,
|
|
54
54
|
public_training_data="https://huggingface.co/datasets/DDSC/nordic-embedding-training-data",
|
|
@@ -59,7 +59,7 @@ qwen_scandi = ModelMeta(
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
mmbert_scandi = ModelMeta(
|
|
62
|
-
loader=sentence_transformers_loader,
|
|
62
|
+
loader=sentence_transformers_loader,
|
|
63
63
|
name="emillykkejensen/mmBERTscandi-base-embedding",
|
|
64
64
|
model_type=["dense"],
|
|
65
65
|
languages=["dan-Latn", "swe-Latn", "nor-Latn", "nob-Latn", "nno-Latn"],
|
|
@@ -72,7 +72,7 @@ mmbert_scandi = ModelMeta(
|
|
|
72
72
|
max_tokens=8192,
|
|
73
73
|
license="apache-2.0",
|
|
74
74
|
reference="https://huggingface.co/emillykkejensen/Qwen3-Embedding-Scandi-0.6B",
|
|
75
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
75
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
76
76
|
use_instructions=True,
|
|
77
77
|
public_training_code=None,
|
|
78
78
|
public_training_data="https://huggingface.co/datasets/DDSC/nordic-embedding-training-data",
|
|
@@ -24,7 +24,7 @@ english_code_retriever = ModelMeta(
|
|
|
24
24
|
max_tokens=8192,
|
|
25
25
|
reference="https://huggingface.co/fyaronskiy/english_code_retriever",
|
|
26
26
|
similarity_fn_name="cosine",
|
|
27
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
27
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
28
28
|
use_instructions=True,
|
|
29
29
|
public_training_code=None,
|
|
30
30
|
public_training_data="https://huggingface.co/datasets/code-search-net/code_search_net",
|
|
@@ -16,7 +16,7 @@ Euler_Legal_Embedding_V1 = ModelMeta(
|
|
|
16
16
|
open_weights=True,
|
|
17
17
|
public_training_code=None,
|
|
18
18
|
public_training_data=None,
|
|
19
|
-
framework=["PyTorch", "Sentence Transformers"],
|
|
19
|
+
framework=["PyTorch", "Sentence Transformers", "safetensors", "Transformers"],
|
|
20
20
|
reference="https://huggingface.co/Mira190/Euler-Legal-Embedding-V1",
|
|
21
21
|
similarity_fn_name="cosine",
|
|
22
22
|
use_instructions=False,
|
|
@@ -24,7 +24,7 @@ Euler_Legal_Embedding_V1 = ModelMeta(
|
|
|
24
24
|
adapted_from="Qwen/Qwen3-Embedding-8B",
|
|
25
25
|
superseded_by=None,
|
|
26
26
|
citation="""@misc{euler2025legal,
|
|
27
|
-
title={Euler-Legal-Embedding: Advanced Legal Representation Learning},
|
|
27
|
+
title={Euler-Legal-Embedding: Advanced Legal Representation Learning},
|
|
28
28
|
author={LawRank Team},
|
|
29
29
|
year={2025},
|
|
30
30
|
publisher={Hugging Face}
|
|
@@ -18,7 +18,7 @@ parsbert = ModelMeta(
|
|
|
18
18
|
max_tokens=512,
|
|
19
19
|
reference="https://huggingface.co/HooshvareLab/bert-base-parsbert-uncased",
|
|
20
20
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
21
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
21
|
+
framework=["Sentence Transformers", "PyTorch", "Transformers"],
|
|
22
22
|
use_instructions=False,
|
|
23
23
|
public_training_code=None,
|
|
24
24
|
public_training_data=None,
|
|
@@ -54,7 +54,7 @@ bert_zwnj = ModelMeta(
|
|
|
54
54
|
max_tokens=512,
|
|
55
55
|
reference="https://huggingface.co/m3hrdadfi/bert-zwnj-wnli-mean-tokens",
|
|
56
56
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
57
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
57
|
+
framework=["Sentence Transformers", "PyTorch", "Transformers"],
|
|
58
58
|
use_instructions=False,
|
|
59
59
|
public_training_code=None,
|
|
60
60
|
public_training_data=None,
|
|
@@ -80,7 +80,7 @@ roberta_zwnj = ModelMeta(
|
|
|
80
80
|
max_tokens=514,
|
|
81
81
|
reference="https://huggingface.co/m3hrdadfi/roberta-zwnj-wnli-mean-tokens",
|
|
82
82
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
83
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
83
|
+
framework=["Sentence Transformers", "PyTorch", "Transformers"],
|
|
84
84
|
use_instructions=False,
|
|
85
85
|
public_training_code=None,
|
|
86
86
|
public_training_data=None,
|
|
@@ -105,7 +105,7 @@ sentence_transformer_parsbert = ModelMeta(
|
|
|
105
105
|
max_tokens=512,
|
|
106
106
|
reference="https://huggingface.co/myrkur/sentence-transformer-parsbert-fa",
|
|
107
107
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
108
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
108
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
109
109
|
use_instructions=False,
|
|
110
110
|
public_training_code=None,
|
|
111
111
|
public_training_data=None,
|
|
@@ -129,7 +129,7 @@ tooka_bert_base = ModelMeta(
|
|
|
129
129
|
max_tokens=512,
|
|
130
130
|
reference="https://huggingface.co/PartAI/TookaBERT-Base",
|
|
131
131
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
132
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
132
|
+
framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
|
|
133
133
|
use_instructions=False,
|
|
134
134
|
public_training_code=None,
|
|
135
135
|
public_training_data=None,
|
|
@@ -156,7 +156,7 @@ tooka_sbert = ModelMeta(
|
|
|
156
156
|
max_tokens=512,
|
|
157
157
|
reference="https://huggingface.co/PartAI/Tooka-SBERT",
|
|
158
158
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
159
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
159
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
160
160
|
use_instructions=False,
|
|
161
161
|
public_training_code=None,
|
|
162
162
|
public_training_data=None,
|
|
@@ -187,7 +187,7 @@ fa_bert = ModelMeta(
|
|
|
187
187
|
max_tokens=512,
|
|
188
188
|
reference="https://huggingface.co/sbunlp/fabert",
|
|
189
189
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
190
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
190
|
+
framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
|
|
191
191
|
use_instructions=False,
|
|
192
192
|
public_training_code=None,
|
|
193
193
|
public_training_data=None,
|
|
@@ -235,7 +235,7 @@ tooka_sbert_v2_small = ModelMeta(
|
|
|
235
235
|
max_tokens=512,
|
|
236
236
|
reference="https://huggingface.co/PartAI/Tooka-SBERT-V2-Small",
|
|
237
237
|
similarity_fn_name="cosine",
|
|
238
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
238
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
239
239
|
use_instructions=False,
|
|
240
240
|
public_training_code=None,
|
|
241
241
|
public_training_data=None,
|
|
@@ -266,7 +266,7 @@ tooka_sbert_v2_large = ModelMeta(
|
|
|
266
266
|
max_tokens=512,
|
|
267
267
|
reference="https://huggingface.co/PartAI/Tooka-SBERT-V2-Large",
|
|
268
268
|
similarity_fn_name="cosine",
|
|
269
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
269
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
270
270
|
use_instructions=False,
|
|
271
271
|
public_training_code=None,
|
|
272
272
|
public_training_data=None,
|
|
@@ -119,7 +119,13 @@ xlmr_base = ModelMeta(
|
|
|
119
119
|
max_tokens=512,
|
|
120
120
|
reference="https://huggingface.co/FacebookAI/xlm-roberta-base",
|
|
121
121
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
122
|
-
framework=[
|
|
122
|
+
framework=[
|
|
123
|
+
"Sentence Transformers",
|
|
124
|
+
"PyTorch",
|
|
125
|
+
"Transformers",
|
|
126
|
+
"ONNX",
|
|
127
|
+
"safetensors",
|
|
128
|
+
],
|
|
123
129
|
use_instructions=False,
|
|
124
130
|
public_training_code=None,
|
|
125
131
|
public_training_data=None,
|
|
@@ -163,7 +169,13 @@ xlmr_large = ModelMeta(
|
|
|
163
169
|
max_tokens=512,
|
|
164
170
|
reference="https://huggingface.co/FacebookAI/xlm-roberta-large",
|
|
165
171
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
166
|
-
framework=[
|
|
172
|
+
framework=[
|
|
173
|
+
"Sentence Transformers",
|
|
174
|
+
"PyTorch",
|
|
175
|
+
"Transformers",
|
|
176
|
+
"ONNX",
|
|
177
|
+
"safetensors",
|
|
178
|
+
],
|
|
167
179
|
use_instructions=False,
|
|
168
180
|
public_training_code=None,
|
|
169
181
|
public_training_data=None,
|
|
@@ -26,7 +26,7 @@ geoembedding = ModelMeta(
|
|
|
26
26
|
max_tokens=32768,
|
|
27
27
|
reference="https://huggingface.co/GeoGPT-Research-Project/GeoEmbedding",
|
|
28
28
|
similarity_fn_name="cosine",
|
|
29
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
29
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
30
30
|
use_instructions=True,
|
|
31
31
|
public_training_code=None,
|
|
32
32
|
public_training_data=None,
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import math
|
|
5
|
+
import warnings
|
|
5
6
|
from typing import TYPE_CHECKING, Any
|
|
6
7
|
|
|
7
8
|
import torch
|
|
@@ -261,9 +262,9 @@ def smart_resize(
|
|
|
261
262
|
w_bar = ceil_by_factor(width * beta, factor)
|
|
262
263
|
|
|
263
264
|
if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO:
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
)
|
|
265
|
+
msg = f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}"
|
|
266
|
+
logger.warning(msg)
|
|
267
|
+
warnings.warn(msg)
|
|
267
268
|
if h_bar > w_bar:
|
|
268
269
|
h_bar = w_bar * MAX_RATIO
|
|
269
270
|
else:
|
|
@@ -359,7 +360,7 @@ gme_qwen2vl_2b = ModelMeta(
|
|
|
359
360
|
max_tokens=32768,
|
|
360
361
|
reference="https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
|
|
361
362
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
362
|
-
framework=["PyTorch"],
|
|
363
|
+
framework=["PyTorch", "Sentence Transformers", "safetensors", "Transformers"],
|
|
363
364
|
use_instructions=True,
|
|
364
365
|
public_training_code=None,
|
|
365
366
|
public_training_data=None,
|
|
@@ -383,7 +384,7 @@ gme_qwen2vl_7b = ModelMeta(
|
|
|
383
384
|
max_tokens=32768,
|
|
384
385
|
reference="https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct",
|
|
385
386
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
386
|
-
framework=["PyTorch"],
|
|
387
|
+
framework=["PyTorch", "Sentence Transformers", "safetensors", "Transformers"],
|
|
387
388
|
use_instructions=True,
|
|
388
389
|
public_training_code=None,
|
|
389
390
|
public_training_data=None,
|
|
@@ -270,7 +270,7 @@ embedding_gemma_300m = ModelMeta(
|
|
|
270
270
|
max_tokens=2048,
|
|
271
271
|
license="gemma",
|
|
272
272
|
reference="https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
|
|
273
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
273
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
274
274
|
use_instructions=True,
|
|
275
275
|
public_training_code=None,
|
|
276
276
|
public_training_data=None,
|
|
@@ -179,7 +179,7 @@ granite_vision_embedding = ModelMeta(
|
|
|
179
179
|
open_weights=True,
|
|
180
180
|
public_training_code=None,
|
|
181
181
|
public_training_data=None,
|
|
182
|
-
framework=["PyTorch"],
|
|
182
|
+
framework=["PyTorch", "Transformers", "safetensors"],
|
|
183
183
|
reference="https://huggingface.co/ibm-granite/granite-vision-3.3-2b-embedding",
|
|
184
184
|
similarity_fn_name="MaxSim",
|
|
185
185
|
use_instructions=True,
|
|
@@ -50,7 +50,7 @@ gritlm7b = ModelMeta(
|
|
|
50
50
|
max_tokens=32768,
|
|
51
51
|
reference="https://huggingface.co/GritLM/GritLM-7B",
|
|
52
52
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
53
|
-
framework=["GritLM", "PyTorch"],
|
|
53
|
+
framework=["GritLM", "PyTorch", "Transformers", "safetensors"],
|
|
54
54
|
use_instructions=True,
|
|
55
55
|
training_datasets=GRIT_LM_TRAINING_DATA,
|
|
56
56
|
# section 3.1 "We finetune our final models from Mistral 7B [68] and Mixtral 8x7B [69] using adaptations of E5 [160] and the Tülu 2 data
|
|
@@ -79,7 +79,7 @@ gritlm8x7b = ModelMeta(
|
|
|
79
79
|
max_tokens=32768,
|
|
80
80
|
reference="https://huggingface.co/GritLM/GritLM-8x7B",
|
|
81
81
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
82
|
-
framework=["GritLM", "PyTorch"],
|
|
82
|
+
framework=["GritLM", "PyTorch", "Transformers", "safetensors"],
|
|
83
83
|
use_instructions=True,
|
|
84
84
|
training_datasets=GRIT_LM_TRAINING_DATA,
|
|
85
85
|
citation=GRITLM_CITATION,
|
|
@@ -53,7 +53,7 @@ gte_qwen2_7b_instruct = ModelMeta(
|
|
|
53
53
|
license="apache-2.0",
|
|
54
54
|
reference="https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct",
|
|
55
55
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
56
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
56
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
57
57
|
use_instructions=True,
|
|
58
58
|
citation=GTE_CITATION,
|
|
59
59
|
public_training_code=None,
|
|
@@ -86,7 +86,7 @@ gte_qwen1_5_7b_instruct = ModelMeta(
|
|
|
86
86
|
max_tokens=32_768,
|
|
87
87
|
reference="https://huggingface.co/Alibaba-NLP/gte-Qwen1.5-7B-instruct",
|
|
88
88
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
89
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
89
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
90
90
|
use_instructions=True,
|
|
91
91
|
public_training_code=None,
|
|
92
92
|
public_training_data=None,
|
|
@@ -123,7 +123,7 @@ gte_qwen2_1_5b_instruct = ModelMeta(
|
|
|
123
123
|
max_tokens=32_768,
|
|
124
124
|
reference="https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct",
|
|
125
125
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
126
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
126
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
127
127
|
use_instructions=True,
|
|
128
128
|
public_training_code=None,
|
|
129
129
|
public_training_data=None,
|
|
@@ -151,7 +151,7 @@ gte_small_zh = ModelMeta(
|
|
|
151
151
|
max_tokens=512,
|
|
152
152
|
reference="https://huggingface.co/thenlper/gte-small-zh",
|
|
153
153
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
154
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
154
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
155
155
|
use_instructions=False,
|
|
156
156
|
public_training_code=None,
|
|
157
157
|
public_training_data=None,
|
|
@@ -179,7 +179,7 @@ gte_base_zh = ModelMeta(
|
|
|
179
179
|
max_tokens=512,
|
|
180
180
|
reference="https://huggingface.co/thenlper/gte-base-zh",
|
|
181
181
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
182
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
182
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
183
183
|
use_instructions=False,
|
|
184
184
|
public_training_code=None,
|
|
185
185
|
public_training_data=None,
|
|
@@ -207,7 +207,7 @@ gte_large_zh = ModelMeta(
|
|
|
207
207
|
max_tokens=512,
|
|
208
208
|
reference="https://huggingface.co/thenlper/gte-large-zh",
|
|
209
209
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
210
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
210
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors"],
|
|
211
211
|
use_instructions=False,
|
|
212
212
|
public_training_code=None,
|
|
213
213
|
public_training_data=None,
|
|
@@ -336,7 +336,7 @@ gte_multilingual_base = ModelMeta(
|
|
|
336
336
|
max_tokens=8192,
|
|
337
337
|
reference="https://huggingface.co/Alibaba-NLP/gte-multilingual-base",
|
|
338
338
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
339
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
339
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
340
340
|
use_instructions=False,
|
|
341
341
|
public_training_code=None,
|
|
342
342
|
public_training_data=None, # couldn't find
|
|
@@ -365,7 +365,13 @@ gte_modernbert_base = ModelMeta(
|
|
|
365
365
|
max_tokens=8192,
|
|
366
366
|
reference="https://huggingface.co/Alibaba-NLP/gte-modernbert-base",
|
|
367
367
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
368
|
-
framework=[
|
|
368
|
+
framework=[
|
|
369
|
+
"Sentence Transformers",
|
|
370
|
+
"PyTorch",
|
|
371
|
+
"Transformers",
|
|
372
|
+
"ONNX",
|
|
373
|
+
"safetensors",
|
|
374
|
+
],
|
|
369
375
|
use_instructions=False,
|
|
370
376
|
public_training_code=None, # couldn't find
|
|
371
377
|
public_training_data=None,
|
|
@@ -402,7 +408,13 @@ gte_base_en_v15 = ModelMeta(
|
|
|
402
408
|
max_tokens=8192,
|
|
403
409
|
reference="https://huggingface.co/Alibaba-NLP/gte-base-en-v1.5",
|
|
404
410
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
405
|
-
framework=[
|
|
411
|
+
framework=[
|
|
412
|
+
"Sentence Transformers",
|
|
413
|
+
"PyTorch",
|
|
414
|
+
"Transformers",
|
|
415
|
+
"ONNX",
|
|
416
|
+
"safetensors",
|
|
417
|
+
],
|
|
406
418
|
use_instructions=False,
|
|
407
419
|
superseded_by=None,
|
|
408
420
|
adapted_from=None,
|
|
@@ -410,21 +422,21 @@ gte_base_en_v15 = ModelMeta(
|
|
|
410
422
|
public_training_data=None,
|
|
411
423
|
training_datasets=None,
|
|
412
424
|
citation="""@misc{zhang2024mgte,
|
|
413
|
-
title={mGTE: Generalized Long-Context Text Representation and Reranking Models for Multilingual Text Retrieval},
|
|
425
|
+
title={mGTE: Generalized Long-Context Text Representation and Reranking Models for Multilingual Text Retrieval},
|
|
414
426
|
author={Xin Zhang and Yanzhao Zhang and Dingkun Long and Wen Xie and Ziqi Dai and Jialong Tang and Huan Lin and Baosong Yang and Pengjun Xie and Fei Huang and Meishan Zhang and Wenjie Li and Min Zhang},
|
|
415
427
|
year={2024},
|
|
416
428
|
eprint={2407.19669},
|
|
417
429
|
archivePrefix={arXiv},
|
|
418
430
|
primaryClass={cs.CL},
|
|
419
|
-
url={https://arxiv.org/abs/2407.19669},
|
|
431
|
+
url={https://arxiv.org/abs/2407.19669},
|
|
420
432
|
}
|
|
421
433
|
@misc{li2023gte,
|
|
422
|
-
title={Towards General Text Embeddings with Multi-stage Contrastive Learning},
|
|
434
|
+
title={Towards General Text Embeddings with Multi-stage Contrastive Learning},
|
|
423
435
|
author={Zehan Li and Xin Zhang and Yanzhao Zhang and Dingkun Long and Pengjun Xie and Meishan Zhang},
|
|
424
436
|
year={2023},
|
|
425
437
|
eprint={2308.03281},
|
|
426
438
|
archivePrefix={arXiv},
|
|
427
439
|
primaryClass={cs.CL},
|
|
428
|
-
url={https://arxiv.org/abs/2308.03281},
|
|
440
|
+
url={https://arxiv.org/abs/2308.03281},
|
|
429
441
|
}""",
|
|
430
442
|
)
|
|
@@ -49,7 +49,7 @@ Hinvec_bidir = ModelMeta(
|
|
|
49
49
|
max_tokens=2048,
|
|
50
50
|
reference="https://huggingface.co/Sailesh97/Hinvec",
|
|
51
51
|
similarity_fn_name="cosine",
|
|
52
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
52
|
+
framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
|
|
53
53
|
use_instructions=True,
|
|
54
54
|
training_datasets=hinvec_training_datasets,
|
|
55
55
|
public_training_code=None,
|
|
@@ -106,7 +106,13 @@ granite_107m_multilingual = ModelMeta(
|
|
|
106
106
|
max_tokens=512,
|
|
107
107
|
reference="https://huggingface.co/ibm-granite/granite-embedding-107m-multilingual",
|
|
108
108
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
109
|
-
framework=[
|
|
109
|
+
framework=[
|
|
110
|
+
"Sentence Transformers",
|
|
111
|
+
"PyTorch",
|
|
112
|
+
"Transformers",
|
|
113
|
+
"ONNX",
|
|
114
|
+
"safetensors",
|
|
115
|
+
],
|
|
110
116
|
adapted_from=None,
|
|
111
117
|
superseded_by=None,
|
|
112
118
|
public_training_code=None,
|
|
@@ -131,7 +137,13 @@ granite_278m_multilingual = ModelMeta(
|
|
|
131
137
|
max_tokens=512,
|
|
132
138
|
reference="https://huggingface.co/ibm-granite/granite-embedding-278m-multilingual",
|
|
133
139
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
134
|
-
framework=[
|
|
140
|
+
framework=[
|
|
141
|
+
"Sentence Transformers",
|
|
142
|
+
"PyTorch",
|
|
143
|
+
"Transformers",
|
|
144
|
+
"ONNX",
|
|
145
|
+
"safetensors",
|
|
146
|
+
],
|
|
135
147
|
adapted_from=None,
|
|
136
148
|
superseded_by=None,
|
|
137
149
|
public_training_code=None,
|
|
@@ -156,7 +168,13 @@ granite_30m_english = ModelMeta(
|
|
|
156
168
|
max_tokens=512,
|
|
157
169
|
reference="https://huggingface.co/ibm-granite/granite-embedding-30m-english",
|
|
158
170
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
159
|
-
framework=[
|
|
171
|
+
framework=[
|
|
172
|
+
"Sentence Transformers",
|
|
173
|
+
"PyTorch",
|
|
174
|
+
"ONNX",
|
|
175
|
+
"safetensors",
|
|
176
|
+
"Transformers",
|
|
177
|
+
],
|
|
160
178
|
adapted_from=None,
|
|
161
179
|
superseded_by=None,
|
|
162
180
|
public_training_code=None,
|
|
@@ -181,7 +199,13 @@ granite_125m_english = ModelMeta(
|
|
|
181
199
|
max_tokens=512,
|
|
182
200
|
reference="https://huggingface.co/ibm-granite/granite-embedding-125m-english",
|
|
183
201
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
184
|
-
framework=[
|
|
202
|
+
framework=[
|
|
203
|
+
"Sentence Transformers",
|
|
204
|
+
"PyTorch",
|
|
205
|
+
"ONNX",
|
|
206
|
+
"safetensors",
|
|
207
|
+
"Transformers",
|
|
208
|
+
],
|
|
185
209
|
adapted_from=None,
|
|
186
210
|
superseded_by=None,
|
|
187
211
|
public_training_code=None,
|
|
@@ -207,7 +231,7 @@ granite_english_r2 = ModelMeta(
|
|
|
207
231
|
max_tokens=8192,
|
|
208
232
|
reference="https://huggingface.co/ibm-granite/granite-embedding-english-r2",
|
|
209
233
|
similarity_fn_name="cosine",
|
|
210
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
234
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
211
235
|
adapted_from=None,
|
|
212
236
|
superseded_by=None,
|
|
213
237
|
public_training_code=None,
|
|
@@ -232,7 +256,7 @@ granite_small_english_r2 = ModelMeta(
|
|
|
232
256
|
max_tokens=8192,
|
|
233
257
|
reference="https://huggingface.co/ibm-granite/granite-embedding-small-english-r2",
|
|
234
258
|
similarity_fn_name="cosine",
|
|
235
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
259
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
236
260
|
adapted_from=None,
|
|
237
261
|
superseded_by=None,
|
|
238
262
|
public_training_code=None,
|
|
@@ -62,7 +62,7 @@ inf_retriever_v1 = ModelMeta(
|
|
|
62
62
|
max_tokens=32768,
|
|
63
63
|
reference="https://huggingface.co/infly/inf-retriever-v1",
|
|
64
64
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
65
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
65
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
66
66
|
use_instructions=True,
|
|
67
67
|
adapted_from="Alibaba-NLP/gte-Qwen2-7B-instruct",
|
|
68
68
|
public_training_code=None,
|
|
@@ -89,7 +89,7 @@ inf_retriever_v1_1_5b = ModelMeta(
|
|
|
89
89
|
max_tokens=32768,
|
|
90
90
|
reference="https://huggingface.co/infly/inf-retriever-v1-1.5b",
|
|
91
91
|
similarity_fn_name=ScoringFunction.COSINE,
|
|
92
|
-
framework=["Sentence Transformers", "PyTorch"],
|
|
92
|
+
framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
|
|
93
93
|
use_instructions=True,
|
|
94
94
|
adapted_from="Alibaba-NLP/gte-Qwen2-1.5B-instruct",
|
|
95
95
|
public_training_code=None,
|