evaluation-embedder 0.1.0__tar.gz

evaluation_embedder-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: evaluation-embedder
+Version: 0.1.0
+Summary: 
+Author: jalal
+Author-email: jalalkhaldi3@gmail.com
+Requires-Python: 3.12.9
+Classifier: Programming Language :: Python :: 3
+Requires-Dist: faiss-cpu (>=1.13.2,<2.0.0)
+Requires-Dist: langchain (>=1.2.3,<2.0.0)
+Requires-Dist: minio (>=7.2.20,<8.0.0)
+Requires-Dist: numpy (>=2.4.1,<3.0.0)
+Requires-Dist: openai (>=2.15.0,<3.0.0)
+Requires-Dist: polars (>=1.37.1,<2.0.0)
+Requires-Dist: pydantic-settings (>=2.12.0,<3.0.0)
+Requires-Dist: qdrant-client (>=1.16.2,<2.0.0)
+Description-Content-Type: text/markdown
+
+

evaluation_embedder-0.1.0/evaluation_embedder/src/constants.py

@@ -0,0 +1,38 @@
+from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar
+
+from polars import Enum
+
+if TYPE_CHECKING:
+    from evaluation_embedder.src.settings import (
+        DatasetSettings,
+        EmbedderSettings,
+        EvaluatorSettings,
+        FromConfigMixinSettings,
+        ProcessorSettings,
+        RetrieverSettings,
+        ScoreSettings,
+        VectorStoreSettings,
+    )
+
+CONFIG_PATH = "/config/config.yaml"
+
+TDataset = TypeVar("TDataset")
+ParquetCompression: TypeAlias = Literal["lz4", "uncompressed", "snappy", "gzip", "brotli", "zstd"]
+TCFromConfigMixin = TypeVar("TCFromConfigMixin", bound="FromConfigMixinSettings")
+TCDataset = TypeVar("TCDataset", bound="DatasetSettings")
+TCEmbedder = TypeVar("TCEmbedder", bound="EmbedderSettings")
+TCEvaluator = TypeVar("TCEvaluator", bound="EvaluatorSettings")
+TCProcessor = TypeVar("TCProcessor", bound="ProcessorSettings")
+TCRetriever = TypeVar("TCRetriever", bound="RetrieverSettings")
+TCScore = TypeVar("TCScore", bound="ScoreSettings")
+TCVectorStore = TypeVar("TCVectorStore", bound="VectorStoreSettings")
+
+
+class EmbeddingPurposeEnum(Enum):
+    DOCUMENT = "document"
+    QUERY = "query"
+
+
+class FAISSIndexType(Enum):
+    FLAT_IP = "flat_ip"
+    FLAT_L2 = "flat_l2"

evaluation_embedder-0.1.0/evaluation_embedder/src/datasets/__init__.py

@@ -0,0 +1,289 @@
+import io
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Self,
+    Tuple,
+    Union,
+    cast,
+    overload,
+)
+
+import polars as pl
+from langchain_core.documents import Document
+from minio import Minio
+from polars._typing import ColumnNameOrSelector, IntoExpr, IntoExprColumn
+
+from evaluation_embedder.src.constants import ParquetCompression, TDataset
+from evaluation_embedder.src.settings import (
+    MinioDatasetSettings,
+    ParquetDatasetSettings,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+class Dataset(ABC, Generic[TDataset]):
+    def __init__(self, service: TDataset):
+        super().__init__()
+        self.service = service
+        self._polars: Optional[pl.DataFrame] = None
+        self._lazy_polars: Optional[pl.LazyFrame] = None
+
+    @classmethod
+    @abstractmethod
+    def from_polars(cls, df: Union[pl.DataFrame, pl.LazyFrame]) -> "Dataset[TDataset]":
+        raise NotImplementedError
+
+    @abstractmethod
+    def to_polars(self) -> pl.DataFrame:
+        raise NotImplementedError
+
+    @abstractmethod
+    def to_lazy_polars(self) -> pl.LazyFrame:
+        raise NotImplementedError
+
+    @classmethod
+    def from_parquet(
+        cls,
+        path: Union[str, Path],
+        *,
+        lazy: bool = True,
+    ) -> "Dataset[TDataset]":
+        df: Union[pl.DataFrame, pl.LazyFrame]
+        if lazy:
+            df = pl.scan_parquet(path)
+        else:
+            df = pl.read_parquet(path)
+
+        return cls.from_polars(df)
+
+    @classmethod
+    def from_minio(
+        cls,
+        *,
+        bucket: str,
+        key: str,
+        endpoint_url: str,
+        access_key: str,
+        secret_key: str,
+    ) -> "Dataset[TDataset]":
+        client = Minio(
+            endpoint_url.replace("http://", "").replace("https://", ""),
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=endpoint_url.startswith("https://"),
+        )
+        response = client.get_object(bucket, key)
+        try:
+            buffer = io.BytesIO(response.read())
+        finally:
+            response.close()
+            response.release_conn()
+
+        df = pl.read_parquet(buffer)
+        return cls.from_polars(df)
+
+    @property
+    def polars(self) -> pl.DataFrame:
+        if self._polars is None:
+            self._polars = self.to_polars()
+        return self._polars
+
+    @property
+    def lazy_polars(self) -> pl.LazyFrame:
+        if self._lazy_polars is None:
+            self._lazy_polars = self.to_lazy_polars()
+        return self._lazy_polars
+
+    @property
+    def polars_shape(self) -> Tuple[int, int]:
+        return self.polars.shape
+
+    @classmethod
+    def from_documents(cls, docs: List[Document]) -> "TextDataset[TDataset]":
+        df = pl.DataFrame(
+            {
+                "page_content": [d.page_content for d in docs],
+                "metadata": [d.metadata or {} for d in docs],
+            }
+        )
+        return cast(TextDataset[TDataset], cls.from_polars(df))
+
+    def with_columns(
+        self,
+        *exprs: IntoExpr | Iterable[IntoExpr],
+        **named_exprs: IntoExpr,
+    ) -> "Dataset[TDataset]":
+        df = self.polars.with_columns(*exprs, **named_exprs)
+        return self.__class__.from_polars(df)
+
+    def filter(
+        self,
+        *predicates: (IntoExprColumn | Iterable[IntoExprColumn] | bool | list[bool]),
+        **constraints: Any,
+    ) -> "Dataset[TDataset]":
+        return self.__class__.from_polars(self.polars.filter(*predicates, **constraints))
+
+    def drop(
+        self,
+        *columns: ColumnNameOrSelector | Iterable[ColumnNameOrSelector],
+        strict: bool = True,
+    ) -> "Dataset[TDataset]":
+        return self.__class__.from_polars(self.polars.drop(*columns, strict=strict))
+
+    def __len__(self) -> int:
+        return self.polars_shape[0]
+
+    @overload
+    def iter_rows(self, *, named: Literal[False] = ..., buffer_size: int = ...) -> Iterator[Tuple[Any, ...]]: ...
+
+    @overload
+    def iter_rows(self, *, named: Literal[True], buffer_size: int = ...) -> Iterator[Dict[str, Any]]: ...
+
+    def iter_rows(
+        self, *, named: Literal[False, True] = False, buffer_size: int = 512
+    ) -> Iterator[Tuple[Any, ...]] | Iterator[Dict[str, Any]]:
+        df_stream = self.lazy_polars.collect(streaming=True)  # type: ignore[call-overload]
+        return df_stream.iter_rows(named=named, buffer_size=buffer_size)  # type: ignore[no-any-return]
+
+
+class TextDataset(Dataset[TDataset], Generic[TDataset]):
+    REQUIRED_COLUMNS = {"page_content", "metadata"}
+
+    def __init__(self, service: TDataset):
+        super().__init__(service)
+        self._validate_schema()
+
+    def _validate_schema(self) -> None:
+        df = self.polars
+        missing = self.REQUIRED_COLUMNS - set(df.columns)
+        if missing:
+            raise ValueError(
+                f"{self.__class__.__name__} requires columns {self.REQUIRED_COLUMNS}, " f"but missing {missing}"
+            )
+
+    def iter_documents(self) -> Iterator[Document]:
+        for row in self.polars.iter_rows(named=True):
+            yield Document(
+                page_content=row["page_content"],
+                metadata=row["metadata"],
+            )
+
+    def dump_documents(
+        self,
+        out_dir: str,
+        prefix: str = "doc",
+        ext: str = ".md",
+        encoding: str = "utf-8",
+    ) -> None:
+        out_dir_path = Path(out_dir)
+        out_dir_path.mkdir(parents=True, exist_ok=True)
+        for i, row in enumerate(self.polars.iter_rows(named=True)):
+            path = out_dir_path / f"{prefix}_{i:05d}{ext}"
+            path.write_text(row["page_content"], encoding=encoding)
+
+    def to_langchain_documents(
+        self,
+    ) -> list[Document]:
+        docs: list[Document] = []
+        for row in self.polars.iter_rows(named=True):
+            docs.append(
+                Document(
+                    page_content=row["page_content"],
+                    metadata=row["metadata"],
+                )
+            )
+        return docs
+
+    def to_minio(
+        self,
+        *,
+        bucket: str,
+        key: str,
+        endpoint_url: str,
+        access_key: str,
+        secret_key: str,
+        compression: ParquetCompression = "zstd",
+        row_group_size: int = 100_000,
+    ) -> None:
+        client = Minio(
+            endpoint_url.replace("http://", "").replace("https://", ""),
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=endpoint_url.startswith("https://"),
+        )
+        buffer = io.BytesIO()
+        self.polars.write_parquet(
+            buffer,
+            compression=compression,
+            row_group_size=row_group_size,
+        )
+        buffer.seek(0)
+        client.put_object(
+            bucket_name=bucket,
+            object_name=key,
+            data=buffer,
+            length=buffer.getbuffer().nbytes,
+            content_type="application/octet-stream",
+        )
+
+    @overload
+    @classmethod
+    def from_config(
+        cls,
+        config: ParquetDatasetSettings,
+    ) -> Self: ...
+
+    @overload
+    @classmethod
+    def from_config(
+        cls,
+        config: MinioDatasetSettings,
+    ) -> Self: ...
+
+    @classmethod
+    def from_config(
+        cls,
+        config: Union[ParquetDatasetSettings, MinioDatasetSettings],
+    ) -> Self:
+        if isinstance(config, ParquetDatasetSettings):
+            df = pl.scan_parquet(config.path) if config.lazy else pl.read_parquet(config.path)
+            return cast(Self, cls.from_polars(df))
+        if isinstance(config, MinioDatasetSettings):
+            return cast(
+                Self,
+                cls.from_minio(
+                    bucket=config.bucket,
+                    key=config.key,
+                    endpoint_url=config.endpoint,
+                    access_key=config.access_key,
+                    secret_key=config.secret_key,
+                ),
+            )
+
+        raise TypeError(f"Unsupported dataset config: {type(config).__name__}")
+
+    @classmethod
+    def from_records(
+        cls,
+        records: List[Tuple[str, Dict[str, Any]]],
+    ) -> "TextDataset[TDataset]":
+        if not records:
+            raise ValueError("records must be non-empty")
+        df = pl.DataFrame(
+            {
+                "page_content": [text for text, _ in records],
+                "metadata": [meta for _, meta in records],
+            }
+        )
+        return cast("TextDataset[TDataset]", cls.from_polars(df))

evaluation_embedder-0.1.0/evaluation_embedder/src/datasets/polars.py

@@ -0,0 +1,26 @@
+from typing import Union, cast
+
+import polars as pl
+
+from evaluation_embedder.src.constants import TDataset
+from evaluation_embedder.src.datasets import TextDataset
+
+
+class PolarsTextDataset(TextDataset[Union[pl.DataFrame, pl.LazyFrame]]):
+
+    def __init__(self, service: Union[pl.DataFrame, pl.LazyFrame]):
+        super().__init__(service)
+
+    @classmethod
+    def from_polars(cls, df: Union[pl.DataFrame, pl.LazyFrame]) -> "TextDataset[TDataset]":
+        return cast("TextDataset[TDataset]", cls(df))
+
+    def to_polars(self) -> pl.DataFrame:
+        if isinstance(self.service, pl.LazyFrame):
+            return self.service.collect()
+        return self.service
+
+    def to_lazy_polars(self) -> pl.LazyFrame:
+        if isinstance(self.service, pl.LazyFrame):
+            return self.service
+        return self.service.lazy()

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/__init__.py

@@ -0,0 +1,175 @@
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Generic, List, Optional, cast
+
+from langchain_core.documents import (
+    Document,  # or: from langchain.schema import Document
+)
+from pydantic import BaseModel, Field
+
+from evaluation_embedder.src.constants import (
+    EmbeddingPurposeEnum,
+    TCEmbedder,
+    TCEvaluator,
+    TCProcessor,
+    TCRetriever,
+    TCScore,
+    TCVectorStore,
+)
+from evaluation_embedder.src.datasets import TextDataset
+from evaluation_embedder.src.mixins import FromConfigMixin
+from evaluation_embedder.src.utils import load_class
+
+_logger = logging.getLogger(__name__)
+
+
+class Score(ABC, Generic[TCScore]):
+
+    class ScoreResult(BaseModel):
+        name: str
+        value: float
+
+    def __init__(self, config: TCScore) -> None:
+        self.config = config
+        _logger.info(f"Initialized Score | class={self.__class__.__name__} | config={config}")
+
+    @abstractmethod
+    def __call__(self, hits: List[bool]) -> ScoreResult:
+        raise NotImplementedError()
+
+
+class Processor(FromConfigMixin[TCProcessor], ABC, Generic[TCProcessor]):
+
+    def __init__(self, config: TCProcessor) -> None:
+        self.config = config
+
+    @abstractmethod
+    def __call__(self, text: str, purpose: EmbeddingPurposeEnum) -> str:
+        raise NotImplementedError()
+
+
+class Embedder(FromConfigMixin[TCEmbedder], ABC, Generic[TCEmbedder]):
+
+    def __init__(self, config: TCEmbedder) -> None:
+        self.config = config
+        _logger.info(f"Initialized Embedder | class={self.__class__.__name__} | config={config}")
+
+    @abstractmethod
+    async def _aembed_documents(self, texts: List[str]) -> List[List[float]]:
+        raise NotImplementedError()
+
+    def process_query(self, text: str, processor: Optional[Processor[Any]]) -> str:
+        if processor:
+            return processor(text=text, purpose=EmbeddingPurposeEnum.QUERY)  # type: ignore[arg-type]
+        return text
+
+    def process_document(self, text: str, processor: Optional[Processor[Any]]) -> str:
+        if processor:
+            return processor(text=text, purpose=EmbeddingPurposeEnum.DOCUMENT)  # type: ignore[arg-type]
+        return text
+
+    async def aembed_documents(
+        self,
+        texts: List[str],
+        processor: Optional[Processor] = None,
+    ) -> List[List[float]]:
+        docs = [self.process_document(t, processor) for t in texts]
+        return await self._aembed_documents(docs)
+
+    async def aembed_query(self, text: str, processor: Optional[Processor] = None) -> List[float]:
+        text = self.process_query(text, processor)
+        return (await self.aembed_documents([text]))[0]
+
+
+class VectorStore(FromConfigMixin[TCVectorStore], ABC, Generic[TCVectorStore]):
+    class ScoredPoint(BaseModel):
+        score: float = Field(..., description="Points vector distance to the query vector")
+        document: Document
+
+    class QueryResponse(BaseModel):
+        points: List["VectorStore.ScoredPoint"]
+
+    def __init__(self, config: TCVectorStore) -> None:
+        self.config = config
+        _logger.info(f"Initialized VectorStore | class={self.__class__.__name__} | config={config}")
+
+    @abstractmethod
+    def query_points(self, query: List[float], *, limit: int) -> QueryResponse:
+        raise NotImplementedError()
+
+
+class Retriever(FromConfigMixin[TCRetriever], ABC, Generic[TCRetriever]):
+
+    def __init__(self, config: TCRetriever) -> None:
+        self.config = config
+
+        _logger.info(f"Initializing Retriever | class={self.__class__.__name__}")
+
+        self.embedder: Embedder[Any] = load_class(self.config.embedder.module_path)(self.config.embedder)
+
+        self.vector_store: VectorStore[Any] = load_class(self.config.vector_store.module_path)(self.config.vector_store)
+
+        self.processor: Processor[Any] = load_class(self.config.processor.module_path)(self.config.processor)
+
+        _logger.info(
+            f"Retriever initialized | embedder={type(self.embedder).__name__} | "
+            f"vector_store={type(self.vector_store).__name__}"
+        )
+
+    async def retrieve(self, query: str, *, limit: int) -> VectorStore.QueryResponse:
+        query_embedding = await self.embedder.aembed_query(query, self.processor)
+        return self.vector_store.query_points(
+            query=query_embedding,
+            limit=limit,
+        )
+
+
+class Evaluator(FromConfigMixin[TCEvaluator], ABC, Generic[TCEvaluator]):
+
+    NB_LOGS_PER_QUERIES = 100
+
+    def __init__(self, config: TCEvaluator) -> None:
+        self.config = config
+
+        _logger.info(f"Initializing Evaluator | class={self.__class__.__name__}")
+
+        self.dataset: TextDataset[Any] = self._load_dataset()
+        self.retriever: Retriever[Any] = load_class(self.config.retriever.module_path)(self.config.retriever)
+        self.scores: List[Score[Any]] = [load_class(s.module_path)(s) for s in self.config.scores]
+
+        _logger.info(
+            f"Evaluator ready | dataset_size={len(self.dataset)} | " f"scores={[type(s).__name__ for s in self.scores]}"
+        )
+
+    def _load_dataset(self) -> TextDataset[Any]:
+        dataset_cls = load_class(self.config.dataset.module_path)
+        _logger.info(
+            f"Instantiating dataset | class={dataset_cls.__name__} | " f"module_path={self.config.dataset.module_path}"
+        )
+        dataset = cast(
+            TextDataset[Any],
+            dataset_cls.from_config(self.config.dataset),
+        )
+        _logger.info(f"Dataset loaded | type={type(dataset).__name__} | " f"size={len(dataset)}")
+        return dataset
+
+    async def eval(self) -> List[List[Score.ScoreResult]]:
+        scores_all = []
+        max_k = max(getattr(score.config, "k", 0) for score in self.scores)
+        _logger.info(f"Starting evaluation | max_k={max_k} | num_queries={len(self.dataset)}")
+        for idx, sample in enumerate(self.dataset.iter_rows(named=True), start=1):
+            scores = []
+            query = sample["metadata"]["query"]
+            page_content = self.retriever.processor(sample["page_content"], purpose=EmbeddingPurposeEnum.DOCUMENT)  # type: ignore[arg-type]
+            response = await self.retriever.retrieve(
+                query,
+                limit=max_k,
+            )
+            hits = [p.document.page_content == page_content for p in response.points]
+            scores = [s(hits) for s in self.scores]
+            scores_all.append(scores)
+            if idx % self.__class__.NB_LOGS_PER_QUERIES == 0 or idx == len(self.dataset):
+                _logger.info(f"Eval progress | processed={idx}/{len(self.dataset)}")
+        _logger.info("Evaluation completed")
+        return scores_all

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/embedders.py

@@ -0,0 +1,27 @@
+from typing import List
+
+from openai import AsyncOpenAI
+
+from evaluation_embedder.src.evaluation import Embedder
+from evaluation_embedder.src.settings import VLLMEmbedderSettings
+
+
+class VLLMEmbedder(Embedder[VLLMEmbedderSettings]):
+    def __init__(self, config: VLLMEmbedderSettings):
+        super().__init__(config)
+        self.client = AsyncOpenAI(
+            base_url=self.config.base_url,
+            api_key="",  # vLLM does not require a key
+        )
+
+    async def _aembed_documents(self, texts: List[str]) -> List[List[float]]:
+        """
+        Async embedding via vLLM OpenAI-compatible API.
+        """
+        response = await self.client.embeddings.create(
+            model=self.config.model_name,
+            input=texts,
+        )
+
+        # Preserve order
+        return [item.embedding for item in response.data]

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/evaluators.py

@@ -0,0 +1,50 @@
+from typing import List, cast
+
+import numpy as np
+from langchain_core.documents import Document
+
+from evaluation_embedder.src.datasets.polars import PolarsTextDataset
+from evaluation_embedder.src.evaluation import Evaluator
+from evaluation_embedder.src.evaluation.vector_stores import FaissVectorStore
+from evaluation_embedder.src.settings import (
+    FaissEvaluatorSettings,
+    QdrantEvaluatorSettings,
+)
+
+
+class QdrantEvaluator(Evaluator[QdrantEvaluatorSettings]):
+    def __init__(self, config: QdrantEvaluatorSettings):
+        super().__init__(config)
+
+
+class FaissEvaluator(Evaluator[FaissEvaluatorSettings]):
+    def __init__(self, config: FaissEvaluatorSettings):
+        super().__init__(config)
+
+    @classmethod
+    async def create(cls, config: FaissEvaluatorSettings) -> "FaissEvaluator":
+        self = cls(config)
+
+        docs = self.get_docs()
+        texts = [d.page_content for d in docs]
+
+        embeddings = np.asarray(
+            await self.retriever.embedder.aembed_documents(texts),
+            dtype="float32",
+        )
+
+        vector_store = cast(FaissVectorStore, self.retriever.vector_store)
+        vector_store.index = vector_store.build_faiss_index(embeddings.shape[-1])
+        vector_store.add_documents(docs, embeddings)
+
+        return self
+
+    def get_docs(self) -> List[Document]:
+        docs_idx = []
+        seen = set()
+        for i, row in enumerate(self.dataset.iter_rows(named=True)):
+            doc_id = row["metadata"]["doc_id"]
+            if doc_id not in seen:
+                seen.add(doc_id)
+                docs_idx.append(i)
+        return PolarsTextDataset(self.dataset.polars[docs_idx]).to_langchain_documents()

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/processors.py

@@ -0,0 +1,16 @@
+from evaluation_embedder.src.constants import EmbeddingPurposeEnum
+from evaluation_embedder.src.evaluation import Processor
+from evaluation_embedder.src.settings import NomicProcessorSettings
+
+
+class NomicProcessor(Processor[NomicProcessorSettings]):
+
+    def __init__(self, config: NomicProcessorSettings) -> None:
+        super().__init__(config)
+
+    def __call__(self, text: str, purpose: EmbeddingPurposeEnum) -> str:
+        if purpose == EmbeddingPurposeEnum.DOCUMENT:
+            return f"search_document: {text}"
+        if purpose == EmbeddingPurposeEnum.QUERY:
+            return f"search_query: {text}"
+        raise ValueError(f"Unsupported embedding purpose {purpose}")

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/retrievers.py

@@ -0,0 +1,26 @@
+from evaluation_embedder.src.evaluation import Retriever
+from evaluation_embedder.src.settings import (
+    FaissVectorStoreSettings,
+    NomicProcessorSettings,
+    QdrantVectorStoreSettings,
+    RetrieverSettings,
+    VLLMEmbedderSettings,
+)
+
+
+class VLLMFAISSRetriever(
+    Retriever[RetrieverSettings[VLLMEmbedderSettings, FaissVectorStoreSettings, NomicProcessorSettings]]
+):
+    def __init__(
+        self, config: RetrieverSettings[VLLMEmbedderSettings, FaissVectorStoreSettings, NomicProcessorSettings]
+    ):
+        super().__init__(config)
+
+
+class VLLMQdrantRetriever(
+    Retriever[RetrieverSettings[VLLMEmbedderSettings, QdrantVectorStoreSettings, NomicProcessorSettings]]
+):
+    def __init__(
+        self, config: RetrieverSettings[VLLMEmbedderSettings, QdrantVectorStoreSettings, NomicProcessorSettings]
+    ):
+        super().__init__(config)

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/scores.py

@@ -0,0 +1,59 @@
+from typing import List
+
+from evaluation_embedder.src.evaluation import Score
+from evaluation_embedder.src.settings import (
+    HitAtKScoreSettings,
+    MRRAtKScoreSettings,
+    PrecisionAtKScoreSettings,
+    RecallAtKScoreSettings,
+)
+
+
+class RecallAtK(Score[RecallAtKScoreSettings]):
+
+    def __call__(self, hits: List[bool]) -> Score.ScoreResult:
+        hits_k = hits[: self.config.k]
+        value = sum(hits_k) / len(hits_k) if hits_k else 0.0
+
+        return Score.ScoreResult(
+            name=f"Recall@{self.config.k}",
+            value=value,
+        )
+
+
+class PrecisionAtK(Score[PrecisionAtKScoreSettings]):
+    def __call__(self, hits: List[bool]) -> Score.ScoreResult:
+        hits_k = hits[: self.config.k]
+        value = sum(hits_k) / self.config.k
+        return Score.ScoreResult(
+            name=f"Precision@{self.config.k}",
+            value=value,
+        )
+
+
+class HitAtK(Score[HitAtKScoreSettings]):
+
+    def __call__(self, hits: List[bool]) -> Score.ScoreResult:
+        hits_k = hits[: self.config.k]
+        value = 1.0 if any(hits_k) else 0.0
+
+        return Score.ScoreResult(
+            name=f"Hit@{self.config.k}",
+            value=value,
+        )
+
+
+class MRRAtK(Score[MRRAtKScoreSettings]):
+
+    def __call__(self, hits: List[bool]) -> Score.ScoreResult:
+        value = 0.0
+
+        for rank, is_hit in enumerate(hits[: self.config.k], start=1):
+            if is_hit:
+                value = 1.0 / rank
+                break
+
+        return Score.ScoreResult(
+            name=f"MRR@{self.config.k}",
+            value=value,
+        )

evaluation_embedder-0.1.0/evaluation_embedder/src/evaluation/vector_stores.py

@@ -0,0 +1,115 @@
+from typing import List, Optional
+
+import faiss  # type:ignore[import-untyped]
+import numpy as np
+from langchain_core.documents import Document
+from qdrant_client import QdrantClient
+
+from evaluation_embedder.src.constants import FAISSIndexType
+from evaluation_embedder.src.evaluation import VectorStore
+from evaluation_embedder.src.evaluation.vector_stores import VectorStore
+from evaluation_embedder.src.settings import (
+    FaissVectorStoreSettings,
+    QdrantVectorStoreSettings,
+)
+
+
+class FaissVectorStore(VectorStore[FaissVectorStoreSettings]):
+
+    def __init__(self, config: FaissVectorStoreSettings):
+        super().__init__(config)
+        self.documents: List[Document] = []
+        self.index: Optional[faiss.Index] = None
+
+    def build_faiss_index(self, dim: int) -> faiss.Index:
+        if self.config.index_type is FAISSIndexType.FLAT_IP:
+            return faiss.IndexFlatIP(dim)
+        return faiss.IndexFlatL2(dim)
+
+    # --------------------------------------------------
+    # Add documents
+    # --------------------------------------------------
+    def add_documents(
+        self,
+        documents: List[Document],
+        embeddings: np.ndarray,
+    ) -> None:
+        if self.config.normalize:
+            faiss.normalize_L2(embeddings)
+        if self.index is None:
+            raise ValueError(f"index should be created before adding documents")
+        self.index.add(embeddings.astype("float32"))
+        self.documents.extend(documents)
+
+    # --------------------------------------------------
+    # Query (same contract as Qdrant)
+    # --------------------------------------------------
+    def query_points(
+        self,
+        query: List[float],
+        *,
+        limit: int,
+    ) -> VectorStore.QueryResponse:
+
+        query_vec = np.asarray(query, dtype="float32")[None, :]
+
+        if self.config.normalize:
+            faiss.normalize_L2(query_vec)
+        if self.index is None:
+            raise ValueError(f"index should be created before querying points")
+        scores, indices = self.index.search(query_vec, limit)
+
+        points: List[VectorStore.ScoredPoint] = []
+
+        for idx, score in zip(indices[0], scores[0]):
+            if idx == -1:
+                continue
+            points.append(
+                VectorStore.ScoredPoint(
+                    score=float(score),
+                    document=self.documents[idx],
+                )
+            )
+        return VectorStore.QueryResponse(points=points)
+
+
+class QdrantVectorStore(VectorStore[QdrantVectorStoreSettings]):
+
+    def __init__(self, config: QdrantVectorStoreSettings):
+        super().__init__(config)
+        self.client = QdrantClient(url=self.config.url)
+
+    def query_points(self, query: List[float], *, limit: int) -> VectorStore.QueryResponse:
+        result = self.client.query_points(
+            collection_name=self.config.collection_name,
+            query=query,
+            limit=limit,
+        )
+        points: List[VectorStore.ScoredPoint] = []
+        for idx, point in enumerate(result.points):
+            payload = point.payload
+            if payload is None:
+                raise ValueError(
+                    f"Qdrant returned a point with no payload | "
+                    f"collection={self.config.collection_name} | "
+                    f"index={idx} | score={point.score}"
+                )
+            if "page_content" not in payload:
+                raise KeyError(
+                    f"Missing 'page_content' in payload | "
+                    f"collection={self.config.collection_name} | "
+                    f"keys={list(payload.keys())}"
+                )
+            page_content = payload["page_content"]
+            metadata = dict(payload["metadata"])
+            points.append(
+                VectorStore.ScoredPoint(
+                    score=point.score,
+                    document=Document(
+                        page_content=page_content,
+                        metadata=metadata,
+                    ),
+                )
+            )
+
+        return VectorStore.QueryResponse(points=points)

evaluation_embedder-0.1.0/evaluation_embedder/src/mixins.py

@@ -0,0 +1,61 @@
+from abc import ABC
+from pathlib import Path
+from typing import Any, Dict, Generic, Self, Type, get_args
+
+import yaml
+
+from evaluation_embedder.src.constants import TCFromConfigMixin
+from evaluation_embedder.src.utils import load_class
+
+
+class FromConfigMixin(ABC, Generic[TCFromConfigMixin]):
+
+    def __init__(self, config: TCFromConfigMixin) -> None:
+        super().__init__()
+        self.config = config
+
+    @classmethod
+    def from_config(
+        cls,
+        config: TCFromConfigMixin,
+    ) -> Self:
+        return cls(config)
+
+    @classmethod
+    def get_config_class(cls) -> Type[TCFromConfigMixin]:
+        return get_args(cls.__orig_bases__[0])[0]  # type: ignore
+
+    @classmethod
+    def from_yaml(
+        cls,
+        path: str,
+        key: str | None = None,
+    ) -> Self:
+        """
+        Load a runtime object from YAML.
+
+        Args:
+            yaml_path: Path to YAML config file
+            settings_cls: Pydantic Settings model to validate config
+            key: Optional top-level YAML key (e.g. "retriever")
+
+        Returns:
+            Instantiated runtime object
+        """
+        yaml_path = Path(path)
+        with yaml_path.open("r") as f:
+            raw: Dict[str, Any] = yaml.safe_load(f)
+        if key is not None:
+            raw = raw[key]
+        settings = cls.get_config_class().model_validate(raw)
+        runtime_cls = load_class(settings.module_path)
+        return runtime_cls(settings)  # type: ignore[no-any-return]
+
+    @classmethod
+    def from_settings(cls) -> Self:
+        """
+        Load runtime object using Pydantic Settings resolution:
+        init > yaml > env > dotenv > secrets
+        """
+        settings = cls.get_config_class()()  # type: ignore[call-arg]
+        return load_class(settings.module_path)(settings)  # type: ignore[no-any-return]

evaluation_embedder-0.1.0/evaluation_embedder/src/settings.py

@@ -0,0 +1,144 @@
+from typing import Generic, List
+
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    YamlConfigSettingsSource,
+)
+
+from evaluation_embedder.src.constants import (
+    CONFIG_PATH,
+    FAISSIndexType,
+    TCDataset,
+    TCEmbedder,
+    TCProcessor,
+    TCRetriever,
+    TCScore,
+    TCVectorStore,
+)
+
+
+class DatasetSettings(BaseSettings):
+    module_path: str
+
+
+class ParquetDatasetSettings(DatasetSettings):
+    path: str
+    lazy: bool
+
+
+class MinioDatasetSettings(DatasetSettings):
+    endpoint: str
+    bucket: str
+    key: str
+    access_key: str
+    secret_key: str
+    model_config = SettingsConfigDict(env_prefix='MINIO_', extra="ignore")
+
+
+class FromConfigMixinSettings(BaseSettings):
+    module_path: str
+    model_config = SettingsConfigDict(
+        yaml_file=CONFIG_PATH,
+        extra="ignore",
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings,
+            YamlConfigSettingsSource(settings_cls),
+            env_settings,
+            dotenv_settings,
+            file_secret_settings,
+        )
+
+
+class EmbedderSettings(FromConfigMixinSettings):
+    model_name: str
+
+
+class VLLMEmbedderSettings(EmbedderSettings):
+    base_url: str
+
+
+class ProcessorSettings(FromConfigMixinSettings):
+    pass
+
+
+class NomicProcessorSettings(ProcessorSettings):
+    pass
+
+
+class ScoreSettings(BaseSettings):
+    module_path: str
+
+
+class RecallAtKScoreSettings(ScoreSettings):
+    k: int
+
+
+class PrecisionAtKScoreSettings(ScoreSettings):
+    k: int
+
+
+class HitAtKScoreSettings(ScoreSettings):
+    k: int
+
+
+class MRRAtKScoreSettings(ScoreSettings):
+    k: int
+
+
+class VectorStoreSettings(FromConfigMixinSettings):
+    pass
+
+
+class QdrantVectorStoreSettings(VectorStoreSettings):
+    url: str
+    collection_name: str
+
+
+class FaissVectorStoreSettings(VectorStoreSettings):
+    index_type: FAISSIndexType
+    normalize: bool
+
+
+class RetrieverSettings(FromConfigMixinSettings, Generic[TCEmbedder, TCVectorStore, TCProcessor]):
+    embedder: TCEmbedder
+    vector_store: TCVectorStore
+    processor: TCProcessor
+
+
+class EvaluatorSettings(FromConfigMixinSettings, Generic[TCDataset, TCRetriever, TCScore]):
+    dataset: TCDataset
+    retriever: TCRetriever
+    scores: List[TCScore]
+
+
+class QdrantEvaluatorSettings(
+    EvaluatorSettings[
+        MinioDatasetSettings,
+        RetrieverSettings[VLLMEmbedderSettings, QdrantVectorStoreSettings, NomicProcessorSettings],
+        RecallAtKScoreSettings,
+    ]
+):
+    pass
+
+
+class FaissEvaluatorSettings(
+    EvaluatorSettings[
+        MinioDatasetSettings,
+        RetrieverSettings[VLLMEmbedderSettings, FaissVectorStoreSettings, NomicProcessorSettings],
+        RecallAtKScoreSettings,
+    ]
+):
+    pass

evaluation_embedder-0.1.0/evaluation_embedder/src/utils.py

@@ -0,0 +1,8 @@
+import importlib
+from typing import Any
+
+
+def load_class(path: str) -> Any:
+    module_path, class_name = path.rsplit(".", 1)
+    module = importlib.import_module(module_path)
+    return getattr(module, class_name)

evaluation_embedder-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "evaluation-embedder"
+version = "0.1.0"
+description = ""
+authors = [
+    {name = "jalal",email = "jalalkhaldi3@gmail.com"}
+]
+readme = "README.md"
+requires-python = "3.12.9"
+dependencies = [
+    "polars (>=1.37.1,<2.0.0)",
+    "minio (>=7.2.20,<8.0.0)",
+    "langchain (>=1.2.3,<2.0.0)",
+    "pydantic-settings (>=2.12.0,<3.0.0)",
+    "openai (>=2.15.0,<3.0.0)",
+    "numpy (>=2.4.1,<3.0.0)",
+    "qdrant-client (>=1.16.2,<2.0.0)",
+    "faiss-cpu (>=1.13.2,<2.0.0)",
+]
+
+
+[dependency-groups]
+notebook = [
+    "jupyter (>=1.1.1,<2.0.0)",
+    "mypy (>=1.19.0,<2.0.0)",
+    "pre-commit (>=4.5.0,<5.0.0)",
+    "types-pyyaml (>=6.0.12.20250915,<7.0.0.0)",
+    "torch (>=2.9.0,<3.0.0)"
+]
+
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"