mteb 2.3.11__py3-none-any.whl → 2.4.2__py3-none-any.whl

mteb/models/model_meta.py

@@ -1,25 +1,46 @@
+from __future__ import annotations
+
+import json
 import logging
+import warnings
 from collections.abc import Callable, Sequence
 from dataclasses import field
 from enum import Enum
+from functools import partial
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal, cast
 
-from huggingface_hub import get_safetensors_metadata
+from huggingface_hub import (
+    GitCommitInfo,
+    ModelCard,
+    ModelCardData,
+    get_safetensors_metadata,
+    hf_hub_download,
+    list_repo_commits,
+    repo_exists,
+)
 from huggingface_hub.errors import (
+    EntryNotFoundError,
     GatedRepoError,
     NotASafetensorsRepoError,
+    RepositoryNotFoundError,
     SafetensorsParsingError,
 )
 from pydantic import BaseModel, ConfigDict, field_validator
+from transformers import AutoConfig
+from typing_extensions import Self
 
+from mteb._helpful_enum import HelpfulStrEnum
 from mteb.languages import check_language_code
+from mteb.models.models_protocols import EncoderProtocol, MTEBModels
 from mteb.types import ISOLanguageScript, Licenses, Modalities, StrDate, StrURL
 
-from .models_protocols import EncoderProtocol, MTEBModels
-
 if TYPE_CHECKING:
+    from sentence_transformers import CrossEncoder, SentenceTransformer
+
     from mteb.abstasks import AbsTask
 
+
 logger = logging.getLogger(__name__)
 
 FRAMEWORKS = Literal[
@@ -37,7 +58,7 @@ FRAMEWORKS = Literal[
 ]
 
 
-class ScoringFunction(str, Enum):
+class ScoringFunction(HelpfulStrEnum):
     """The scoring function used by the models."""
 
     COSINE = "cosine"
@@ -58,6 +79,9 @@ def _get_loader_name(
     return loader.__name__
 
 
+_SENTENCE_TRANSFORMER_LIB_NAME = "Sentence Transformers"
+
+
 class ModelMeta(BaseModel):
     """The model metadata object.
 
@@ -72,7 +96,7 @@ class ModelMeta(BaseModel):
             models).
         embed_dim: The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings.
         revision: The revision number of the model. If None, it is assumed that the metadata (including the loader) is valid for all revisions of the model.
-        release_date: The date the model's revision was released.
+        release_date: The date the model's revision was released. If None, then release date will be added based on 1st commit in hf repository of model.
         license: The license under which the model is released. Required if open_weights is True.
         open_weights: Whether the model is open source or proprietary.
         public_training_code: A link to the publicly available training code. If None, it is assumed that the training code is not publicly available.
@@ -212,9 +236,198 @@ class ModelMeta(BaseModel):
             raise ValueError("Model name is not set")
         return self.name.replace("/", "__").replace(" ", "_")
 
-    def is_zero_shot_on(
-        self, tasks: Sequence["AbsTask"] | Sequence[str]
-    ) -> bool | None:
+    @classmethod
+    def _from_hub(
+        cls,
+        model_name: str | None,
+        revision: str | None = None,
+        compute_metadata: bool = True,
+    ) -> Self:
+        """Generates a ModelMeta from a HuggingFace model name.
+
+        Args:
+            model_name: The HuggingFace model name.
+            revision: Revision of the model
+            compute_metadata: Add metadata based on model card
+
+        Returns:
+            The generated ModelMeta.
+        """
+        from mteb.models import sentence_transformers_loader
+
+        loader = sentence_transformers_loader
+        frameworks: list[FRAMEWORKS] = ["PyTorch"]
+        model_license = None
+        reference = None
+        n_parameters = None
+        memory_usage_mb = None
+        release_date = None
+        embedding_dim = None
+        max_tokens = None
+
+        if model_name and compute_metadata and repo_exists(model_name):
+            reference = "https://huggingface.co/" + model_name
+            card = ModelCard.load(model_name)
+            card_data: ModelCardData = card.data
+            try:
+                model_config = AutoConfig.from_pretrained(model_name)
+            except Exception as e:
+                # some models can't load AutoConfig (e.g. `average_word_embeddings_levy_dependency`)
+                model_config = None
+                logger.warning(f"Can't get configuration for {model_name}. Error: {e}")
+
+            if (
+                card_data.library_name == _SENTENCE_TRANSFORMER_LIB_NAME
+                or _SENTENCE_TRANSFORMER_LIB_NAME in card_data.tags
+            ):
+                frameworks.append(_SENTENCE_TRANSFORMER_LIB_NAME)
+            else:
+                msg = "Model library not recognized, defaulting to Sentence Transformers loader."
+                logger.warning(msg)
+                warnings.warn(msg)
+
+            if revision is None:
+                revisions = _get_repo_commits(model_name, "model")
+                revision = revisions[0].commit_id if revisions else None
+
+            release_date = cls.fetch_release_date(model_name)
+            model_license = card_data.license
+            n_parameters = cls._calculate_num_parameters_from_hub(model_name)
+            memory_usage_mb = cls._calculate_memory_usage_mb(model_name, n_parameters)
+            if model_config and hasattr(model_config, "hidden_size"):
+                embedding_dim = model_config.hidden_size
+            if model_config and hasattr(model_config, "max_position_embeddings"):
+                max_tokens = model_config.max_position_embeddings
+
+        return cls(
+            loader=loader,
+            name=model_name or "no_model_name/available",
+            revision=revision or "no_revision_available",
+            reference=reference,
+            release_date=release_date,
+            languages=None,
+            license=model_license,
+            framework=frameworks,
+            training_datasets=None,
+            similarity_fn_name=None,
+            n_parameters=n_parameters,
+            memory_usage_mb=memory_usage_mb,
+            max_tokens=max_tokens,
+            embed_dim=embedding_dim,
+            open_weights=True,
+            public_training_code=None,
+            public_training_data=None,
+            use_instructions=None,
+            modalities=[],
+        )
+
+    @classmethod
+    def from_sentence_transformer_model(
+        cls,
+        model: SentenceTransformer,
+        revision: str | None = None,
+        compute_metadata: bool = True,
+    ) -> Self:
+        """Generates a ModelMeta from a SentenceTransformer model.
+
+        Args:
+            model: SentenceTransformer model.
+            revision: Revision of the model
+            compute_metadata: Add metadata based on model card
+
+        Returns:
+            The generated ModelMeta.
+        """
+        name: str | None = (
+            model.model_card_data.model_name
+            if model.model_card_data.model_name
+            else model.model_card_data.base_model
+        )
+        meta = cls._from_hub(name, revision, compute_metadata)
+        if _SENTENCE_TRANSFORMER_LIB_NAME not in meta.framework:
+            meta.framework.append("Sentence Transformers")
+        meta.revision = model.model_card_data.base_model_revision or meta.revision
+        meta.max_tokens = model.max_seq_length
+        meta.embed_dim = model.get_sentence_embedding_dimension()
+        meta.similarity_fn_name = ScoringFunction.from_str(model.similarity_fn_name)
+        meta.modalities = ["text"]
+        return meta
+
+    @classmethod
+    def from_hub(
+        cls,
+        model: str,
+        revision: str | None = None,
+        compute_metadata: bool = True,
+    ) -> Self:
+        """Generates a ModelMeta for model from HuggingFace hub.
+
+        Args:
+            model: Name of the model from HuggingFace hub. For example, `intfloat/multilingual-e5-large`
+            revision: Revision of the model
+            compute_metadata: Add metadata based on model card
+
+        Returns:
+            The generated ModelMeta.
+        """
+        meta = cls._from_hub(model, revision, compute_metadata)
+        if _SENTENCE_TRANSFORMER_LIB_NAME not in meta.framework:
+            meta.framework.append("Sentence Transformers")
+        meta.modalities = ["text"]
+
+        if model and compute_metadata and repo_exists(model):
+            # have max_seq_length field
+            sbert_config = _get_json_from_hub(
+                model, "sentence_bert_config.json", "model", revision=revision
+            )
+            if sbert_config:
+                meta.max_tokens = (
+                    sbert_config.get("max_seq_length", None) or meta.max_tokens
+                )
+            # have model type, similarity function fields
+            config_sbert = _get_json_from_hub(
+                model, "config_sentence_transformers.json", "model", revision=revision
+            )
+            if (
+                config_sbert is not None
+                and config_sbert.get("similarity_fn_name") is not None
+            ):
+                meta.similarity_fn_name = ScoringFunction.from_str(
+                    config_sbert.get("similarity_fn_name")
+                )
+            else:
+                meta.similarity_fn_name = ScoringFunction.COSINE
+        return meta
+
+    @classmethod
+    def from_cross_encoder(
+        cls,
+        model: CrossEncoder,
+        revision: str | None = None,
+        compute_metadata: bool = True,
+    ) -> Self:
+        """Generates a ModelMeta from a CrossEncoder.
+
+        Args:
+            model: The CrossEncoder model
+            revision: Revision of the model
+            compute_metadata: Add metadata based on model card
+
+        Returns:
+            The generated ModelMeta
+        """
+        from mteb.models import CrossEncoderWrapper
+
+        meta = cls._from_hub(model.model.name_or_path, revision, compute_metadata)
+        if _SENTENCE_TRANSFORMER_LIB_NAME not in meta.framework:
+            meta.framework.append("Sentence Transformers")
+        meta.revision = model.config._commit_hash or meta.revision
+        meta.loader = CrossEncoderWrapper
+        meta.embed_dim = None
+        meta.modalities = ["text"]
+        return meta
+
+    def is_zero_shot_on(self, tasks: Sequence[AbsTask] | Sequence[str]) -> bool | None:
         """Indicates whether the given model can be considered zero-shot or not on the given tasks.
 
         Returns:
@@ -267,7 +480,7 @@ class ModelMeta(BaseModel):
         return return_dataset
 
     def zero_shot_percentage(
-        self, tasks: Sequence["AbsTask"] | Sequence[str]
+        self, tasks: Sequence[AbsTask] | Sequence[str]
     ) -> int | None:
         """Indicates how out-of-domain the selected tasks are for the given model.
 
@@ -290,18 +503,38 @@ class ModelMeta(BaseModel):
         perc_overlap = 100 * (len(overlap) / len(benchmark_datasets))
         return int(100 - perc_overlap)
 
-    def calculate_memory_usage_mb(self) -> int | None:
-        """Calculates the memory usage (in FP32) of the model in MB.
+    @staticmethod
+    def _calculate_num_parameters_from_hub(model_name: str | None = None) -> int | None:
+        try:
+            safetensors_metadata = get_safetensors_metadata(model_name)
+            if len(safetensors_metadata.parameter_count) >= 0:
+                return sum(safetensors_metadata.parameter_count.values())
+        except (
+            NotASafetensorsRepoError,
+            SafetensorsParsingError,
+            GatedRepoError,
+            RepositoryNotFoundError,
+        ) as e:
+            logger.warning(
+                f"Can't calculate number of parameters for {model_name}. Got error {e}"
+            )
+            return None
+
+    def calculate_num_parameters_from_hub(self) -> int | None:
+        """Calculates the number of parameters in the model.
 
         Returns:
-            The memory usage of the model in MB, or None if it cannot be determined.
+            Number of parameters in the model.
         """
-        if "API" in self.framework:
-            return None
+        return self._calculate_num_parameters_from_hub(self.name)
 
+    @staticmethod
+    def _calculate_memory_usage_mb(
+        model_name: str, n_parameters: int | None
+    ) -> int | None:
         MB = 1024**2  # noqa: N806
         try:
-            safetensors_metadata = get_safetensors_metadata(self.name)  # type: ignore
+            safetensors_metadata = get_safetensors_metadata(model_name)
             if len(safetensors_metadata.parameter_count) >= 0:
                 dtype_size_map = {
                     "F64": 8,  # 64-bit float
@@ -320,18 +553,130 @@ class ModelMeta(BaseModel):
                     for dtype, parameters in safetensors_metadata.parameter_count.items()
                 )
                 return round(total_memory_bytes / MB)  # Convert to MB
+        except (
+            NotASafetensorsRepoError,
+            SafetensorsParsingError,
+            GatedRepoError,
+            RepositoryNotFoundError,
+        ) as e:
+            logger.warning(
+                f"Can't calculate memory usage for {model_name}. Got error {e}"
+            )
 
-        except (NotASafetensorsRepoError, SafetensorsParsingError, GatedRepoError):
-            pass
-        if self.n_parameters is None:
+        if n_parameters is None:
             return None
         # Model memory in bytes. For FP32 each parameter is 4 bytes.
-        model_memory_bytes = self.n_parameters * 4
+        model_memory_bytes = n_parameters * 4
 
         # Convert to MB
         model_memory_mb = model_memory_bytes / MB
         return round(model_memory_mb)
 
+    def calculate_memory_usage_mb(self) -> int | None:
+        """Calculates the memory usage of the model in MB.
+
+        Returns:
+            The memory usage of the model in MB, or None if it cannot be determined.
+        """
+        if "API" in self.framework or self.name is None:
+            return None
+
+        return self._calculate_memory_usage_mb(self.model_name, self.n_parameters)
+
+    @staticmethod
+    def fetch_release_date(model_name: str) -> StrDate | None:
+        """Fetches the release date from HuggingFace Hub based on the first commit.
+
+        Returns:
+            The release date in YYYY-MM-DD format, or None if it cannot be determined.
+        """
+        commits = _get_repo_commits(repo_id=model_name, repo_type="model")
+        if commits:
+            initial_commit = commits[-1]
+            release_date = initial_commit.created_at.strftime("%Y-%m-%d")
+            return release_date
+        return None
+
+    def to_python(self) -> str:
+        """Returns a string representation of the model."""
+        return _pydantic_instance_to_code(self)
+
+
+def _pydantic_instance_to_code(
+    model: BaseModel,
+    indent: int = 4,
+    *,
+    only_set_fields: bool = False,
+) -> str:
+    """Convert a Pydantic model instance into valid Python constructor code.
+
+    If only_set_fields=True, only fields explicitly provided at model construction
+    time are printed (i.e., excludes fields that came only from defaults).
+
+    Arguments:
+        model: The Pydantic model to convert.
+        indent: The indentation to use.
+        only_set_fields: If True, only fields explicitly provided at model construction time
+    """
+    cls_name = model.__class__.__name__
+    pad = " " * indent
+    lines: list[str] = [f"{cls_name}("]
+
+    model_fields = list(type(model).model_fields.keys())
+
+    if only_set_fields:
+        field_names = [n for n in model_fields if n in model.model_fields_set]
+    else:
+        field_names = model_fields
+
+    for field_name in field_names:
+        value = getattr(model, field_name)
+        value_code = _value_to_code(value, indent)
+        lines.append(f"{pad}{field_name}={value_code},")
+
+    lines.append(")")
+    return "\n".join(lines)
+
+
+def _value_to_code(value: Any, indent: int) -> str:
+    """Convert a Python value into valid Python source code."""
+    if isinstance(value, BaseModel):
+        return _pydantic_instance_to_code(value, indent, only_set_fields=True)
+
+    if callable(value):
+        if isinstance(value, partial):
+            return value.func.__name__
+        return value.__name__
+
+    if isinstance(value, Enum):
+        return f"{value.__class__.__name__}.{value.name}"
+
+    if isinstance(value, str):
+        return repr(value)
+
+    if isinstance(value, list):
+        if not value:
+            return "[]"
+        inner = ", ".join(_value_to_code(v, indent) for v in value)
+        return f"[{inner}]"
+
+    if isinstance(value, set):
+        if not value:
+            return "set()"
+        inner = ", ".join(_value_to_code(v, indent) for v in sorted(value))
+        return f"{{{inner}}}"
+
+    if isinstance(value, dict):
+        if not value:
+            return "{}"
+        inner = ", ".join(
+            f"{_value_to_code(k, indent)}: {_value_to_code(v, indent)}"
+            for k, v in value.items()
+        )
+        return f"{{{inner}}}"
+
+    return repr(value)
+
 
 def _collect_similar_tasks(dataset: str, visited: set[str]) -> set[str]:
     """Recursively collect all similar tasks for a given dataset.
@@ -364,3 +709,35 @@ def _collect_similar_tasks(dataset: str, visited: set[str]) -> set[str]:
             similar.update(_collect_similar_tasks(parent, visited))
 
     return similar
+
+
+def _get_repo_commits(repo_id: str, repo_type: str) -> list[GitCommitInfo] | None:
+    try:
+        return list_repo_commits(repo_id=repo_id, repo_type=repo_type)
+    except (GatedRepoError, RepositoryNotFoundError) as e:
+        logger.warning(f"Can't get commits of {repo_id}: {e}")
+        return None
+
+
+def _get_json_from_hub(
+    repo_id: str, file_name: str, repo_type: str, revision: str | None = None
+) -> dict[str, Any] | None:
+    path = _get_file_on_hub(repo_id, file_name, repo_type, revision)
+    if path is None:
+        return None
+
+    with Path(path).open() as f:
+        js = json.load(f)
+    return js
+
+
+def _get_file_on_hub(
+    repo_id: str, file_name: str, repo_type: str, revision: str | None = None
+) -> str | None:
+    try:
+        return hf_hub_download(
+            repo_id=repo_id, filename=file_name, repo_type=repo_type, revision=revision
+        )
+    except (GatedRepoError, RepositoryNotFoundError, EntryNotFoundError) as e:
+        logger.warning(f"Can't get file {file_name} of {repo_id}: {e}")
+        return None

mteb/models/sentence_transformer_wrapper.py

@@ -68,11 +68,8 @@ class SentenceTransformerEncoderWrapper(AbsEncoder):
             self.model = SentenceTransformer(model, revision=revision, **kwargs)
         else:
             self.model = model
-        from mteb.models.get_model_meta import (
-            _model_meta_from_sentence_transformers,
-        )
 
-        self.mteb_model_meta = _model_meta_from_sentence_transformers(self.model)
+        self.mteb_model_meta = ModelMeta.from_sentence_transformer_model(self.model)
 
         built_in_prompts = getattr(self.model, "prompts", None)
         if built_in_prompts and not model_prompts:
@@ -268,14 +265,12 @@ class CrossEncoderWrapper:
     ) -> None:
         from sentence_transformers import CrossEncoder
 
-        from mteb.models.get_model_meta import _model_meta_from_cross_encoder
-
         if isinstance(model, CrossEncoder):
             self.model = model
         elif isinstance(model, str):
             self.model = CrossEncoder(model, revision=revision, **kwargs)
 
-        self.mteb_model_meta = _model_meta_from_cross_encoder(self.model)
+        self.mteb_model_meta = ModelMeta.from_cross_encoder(self.model)
 
     def predict(
         self,

mteb/tasks/reranking/jpn/__init__.py

@@ -1,5 +1,13 @@
 from .j_qa_ra_reranking import JQaRAReranking
+from .j_qa_ra_reranking_lite import JQaRARerankingLite
 from .ja_cwir_reranking import JaCWIRReranking
+from .ja_cwir_reranking_lite import JaCWIRRerankingLite
 from .m_marco_reranking import VoyageMMarcoReranking
 
-__all__ = ["JQaRAReranking", "JaCWIRReranking", "VoyageMMarcoReranking"]
+__all__ = [
+    "JQaRAReranking",
+    "JQaRARerankingLite",
+    "JaCWIRReranking",
+    "JaCWIRRerankingLite",
+    "VoyageMMarcoReranking",
+]

mteb/tasks/reranking/jpn/j_qa_ra_reranking_lite.py

@@ -0,0 +1,49 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JQaRARerankingLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JQaRARerankingLite",
+        dataset={
+            "path": "mteb/JQaRARerankingLite",
+            "revision": "d23d3ad479f74824ed126052e810eac47e685558",
+        },
+        description=(
+            "JQaRA (Japanese Question Answering with Retrieval Augmentation) is a reranking dataset "
+            "consisting of questions from JAQKET and corpus from Japanese Wikipedia. This is the lightweight "
+            "version with a reduced corpus (172,897 documents) constructed using hard negatives from "
+            "5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/hotchpotch/JQaRA",
+        type="Reranking",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2020-01-01", "2025-01-01"),
+        domains=["Encyclopaedic", "Non-fiction", "Written"],
+        task_subtypes=["Question answering"],
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=["jpn-Jpan"],
+        sample_creation="found",
+        adapted_from=["JQaRAReranking"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@misc{yuichi-tateno-2024-jqara,
+  author = {Yuichi Tateno},
+  title = {JQaRA: Japanese Question Answering with Retrieval Augmentation
+- 検索拡張(RAG)評価のための日本語Q&Aデータセット},
+  url = {https://huggingface.co/datasets/hotchpotch/JQaRA},
+}
+""",
+    )

mteb/tasks/reranking/jpn/ja_cwir_reranking_lite.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JaCWIRRerankingLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JaCWIRRerankingLite",
+        dataset={
+            "path": "mteb/JaCWIRRerankingLite",
+            "revision": "b7c738193fb9b20c97c2b5d9a8fa3f3d28503dc0",
+        },
+        description=(
+            "JaCWIR (Japanese Casual Web IR) is a dataset consisting of questions and webpage meta descriptions "
+            "collected from Hatena Bookmark. This is the lightweight reranking version with a reduced corpus "
+            "(188,033 documents) constructed using hard negatives from 5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
+        type="Reranking",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2020-01-01", "2025-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["JaCWIRReranking"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@misc{yuichi-tateno-2024-jacwir,
+  author = {Yuichi Tateno},
+  title = {JaCWIR: Japanese Casual Web IR - 日本語情報検索評価のための小規模でカジュアルなWebタイトルと概要のデータセット},
+  url = {https://huggingface.co/datasets/hotchpotch/JaCWIR},
+}
+""",
+    )

mteb/tasks/retrieval/code/fresh_stack_retrieval.py

@@ -25,11 +25,14 @@ class FreshStackRetrieval(AbsTaskRetrieval):
         dialect=[],
         sample_creation="found",
         bibtex_citation=r"""
-@article{freshstack2023,
-  author = {FreshStack Authors},
-  journal = {arXiv preprint arXiv:2301.12345},
-  title = {FreshStack: A Multi-language Code Generation and Retrieval Benchmark},
-  year = {2023},
+@misc{thakur2025freshstackbuildingrealisticbenchmarks,
+  archiveprefix = {arXiv},
+  author = {Nandan Thakur and Jimmy Lin and Sam Havens and Michael Carbin and Omar Khattab and Andrew Drozdov},
+  eprint = {2504.13128},
+  primaryclass = {cs.IR},
+  title = {FreshStack: Building Realistic Benchmarks for Evaluating Retrieval on Technical Documents},
+  url = {https://arxiv.org/abs/2504.13128},
+  year = {2025},
 }
 """,
     )

mteb/tasks/retrieval/jpn/__init__.py

@@ -1,8 +1,12 @@
 from .ja_cwir_retrieval import JaCWIRRetrieval
+from .ja_cwir_retrieval_lite import JaCWIRRetrievalLite
 from .ja_gov_faqs_retrieval import JaGovFaqsRetrieval
 from .ja_qu_ad_retrieval import JaQuADRetrieval
 from .japanese_legal1_retrieval import JapaneseLegal1Retrieval
 from .jaqket_retrieval import JaqketRetrieval
+from .jaqket_retrieval_lite import JaqketRetrievalLite
+from .miracl_ja_retrieval_lite import MIRACLJaRetrievalLite
+from .mr_tydi_ja_retrieval_lite import MrTyDiJaRetrievalLite
 from .nlp_journal_abs_article_retrieval import (
     NLPJournalAbsArticleRetrieval,
     NLPJournalAbsArticleRetrievalV2,
@@ -22,10 +26,14 @@ from .nlp_journal_title_intro_retrieval import (
 
 __all__ = [
     "JaCWIRRetrieval",
+    "JaCWIRRetrievalLite",
     "JaGovFaqsRetrieval",
     "JaQuADRetrieval",
     "JapaneseLegal1Retrieval",
     "JaqketRetrieval",
+    "JaqketRetrievalLite",
+    "MIRACLJaRetrievalLite",
+    "MrTyDiJaRetrievalLite",
     "NLPJournalAbsArticleRetrieval",
     "NLPJournalAbsArticleRetrievalV2",
     "NLPJournalAbsIntroRetrieval",