evret 0.0.1b0__py3-none-any.whl

evret/__init__.py

@@ -0,0 +1,53 @@
+"""Evret package."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from evret.evaluation import (
+    DocumentExample,
+    EvaluationDataset,
+    EvaluationResults,
+    Evaluator,
+    QueryExample,
+)
+from evret.errors import EvretError, EvretValidationError, OptionalDependencyError
+from evret.integrations import LangChainRetrieverAdapter, LlamaIndexRetrieverAdapter
+from evret.metrics import AveragePrecision, HitRate, MRR, NDCG, Precision, Recall
+from evret.retrievers import (
+    BaseRetriever,
+    ChromaRetriever,
+    MilvusRetriever,
+    QdrantRetriever,
+    RetrievalResult,
+    WeaviateRetriever,
+)
+
+try:
+    __version__ = version("evret")
+except PackageNotFoundError:  # pragma: no cover - only when package metadata is unavailable
+    __version__ = "0.0.0"
+
+__all__ = [
+    "__version__",
+    "AveragePrecision",
+    "BaseRetriever",
+    "ChromaRetriever",
+    "DocumentExample",
+    "EvaluationDataset",
+    "EvaluationResults",
+    "Evaluator",
+    "EvretError",
+    "EvretValidationError",
+    "HitRate",
+    "LangChainRetrieverAdapter",
+    "LlamaIndexRetrieverAdapter",
+    "MilvusRetriever",
+    "MRR",
+    "NDCG",
+    "OptionalDependencyError",
+    "Precision",
+    "QueryExample",
+    "QdrantRetriever",
+    "Recall",
+    "RetrievalResult",
+    "WeaviateRetriever",
+]

evret/errors.py

@@ -0,0 +1,15 @@
+"""Evret exception hierarchy."""
+
+from __future__ import annotations
+
+
+class EvretError(Exception):
+    """Base exception for Evret."""
+
+
+class EvretValidationError(ValueError, EvretError):
+    """Raised when user input or data format is invalid."""
+
+
+class OptionalDependencyError(ImportError, EvretError):
+    """Raised when an optional dependency required by a feature is missing."""

evret/evaluation/__init__.py

@@ -0,0 +1,13 @@
+"""Evaluation exports."""
+
+from evret.evaluation.dataset import DocumentExample, EvaluationDataset, QueryExample
+from evret.evaluation.evaluator import Evaluator
+from evret.evaluation.results import EvaluationResults
+
+__all__ = [
+    "QueryExample",
+    "DocumentExample",
+    "EvaluationDataset",
+    "Evaluator",
+    "EvaluationResults",
+]

evret/evaluation/dataset.py

@@ -0,0 +1,139 @@
+"""Evaluation dataset models and loaders."""
+
+from __future__ import annotations
+
+import csv
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from evret.errors import EvretValidationError
+from evret.utils import (
+    normalize_unique_non_empty_strings,
+    require_file_exists,
+    require_non_empty_str,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class QueryExample:
+    """One query item in an evaluation dataset."""
+
+    query_id: str
+    query_text: str
+    relevant_doc_ids: list[str]
+
+
+@dataclass(frozen=True, slots=True)
+class DocumentExample:
+    """One document entry in an evaluation dataset."""
+
+    doc_id: str
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class EvaluationDataset:
+    """Evaluation dataset containing query examples and optional documents."""
+
+    queries: list[QueryExample]
+    documents: list[DocumentExample] = field(default_factory=list)
+
+    @classmethod
+    def from_json(cls, path: str | Path) -> EvaluationDataset:
+        dataset_path = require_file_exists(path, "dataset")
+        raw_data = json.loads(dataset_path.read_text(encoding="utf-8"))
+        if not isinstance(raw_data, dict):
+            raise EvretValidationError("root JSON value must be an object")
+
+        raw_queries = raw_data.get("queries", [])
+        raw_documents = raw_data.get("documents", [])
+        if not isinstance(raw_queries, list):
+            raise EvretValidationError("queries must be a list")
+        if not isinstance(raw_documents, list):
+            raise EvretValidationError("documents must be a list")
+
+        queries = [cls._parse_query_item(item) for item in raw_queries]
+        documents = [cls._parse_document_item(item) for item in raw_documents]
+        return cls(queries=queries, documents=documents)
+
+    @classmethod
+    def from_csv(cls, path: str | Path) -> EvaluationDataset:
+        dataset_path = require_file_exists(path, "dataset")
+        query_items: list[QueryExample] = []
+        with dataset_path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            if reader.fieldnames is None:
+                raise EvretValidationError("CSV must include a header row")
+            for row_index, row in enumerate(reader, start=1):
+                query_text = (row.get("query_text") or row.get("query") or "").strip()
+                if not query_text:
+                    raise EvretValidationError(f"query_text is required in row {row_index}")
+
+                query_id = (
+                    row.get("query_id")
+                    or row.get("id")
+                    or f"q{row_index}"
+                )
+                query_id = require_non_empty_str(query_id, "query_id")
+                relevant_doc_ids = cls._parse_relevant_docs(row.get("relevant_docs", ""))
+                query_items.append(
+                    QueryExample(
+                        query_id=query_id,
+                        query_text=query_text,
+                        relevant_doc_ids=relevant_doc_ids,
+                    )
+                )
+
+        return cls(queries=query_items, documents=[])
+
+    @staticmethod
+    def _parse_query_item(item: Any) -> QueryExample:
+        if not isinstance(item, dict):
+            raise EvretValidationError("each query item must be an object")
+
+        query_id_raw = item.get("query_id") or item.get("id")
+        query_text_raw = item.get("query_text") or item.get("query")
+        relevant_docs_raw = item.get("relevant_docs") or item.get("relevant_doc_ids") or []
+
+        query_id = require_non_empty_str(query_id_raw or "", "query_id")
+        query_text = require_non_empty_str(query_text_raw or "", "query_text")
+        if not isinstance(relevant_docs_raw, list):
+            raise EvretValidationError("relevant_docs must be a list")
+
+        relevant_doc_ids = normalize_unique_non_empty_strings(relevant_docs_raw)
+        return QueryExample(
+            query_id=query_id,
+            query_text=query_text,
+            relevant_doc_ids=relevant_doc_ids,
+        )
+
+    @staticmethod
+    def _parse_document_item(item: Any) -> DocumentExample:
+        if not isinstance(item, dict):
+            raise EvretValidationError("each document item must be an object")
+
+        doc_id = require_non_empty_str(item.get("doc_id") or "", "doc_id")
+        text = str(item.get("text") or "")
+        metadata = item.get("metadata") or {}
+        if not isinstance(metadata, dict):
+            raise EvretValidationError("document metadata must be an object")
+
+        return DocumentExample(doc_id=doc_id, text=text, metadata=dict(metadata))
+
+    @staticmethod
+    def _parse_relevant_docs(raw_value: str) -> list[str]:
+        value = raw_value.strip()
+        if not value:
+            return []
+
+        try:
+            loaded = json.loads(value)
+        except json.JSONDecodeError:
+            loaded = [part.strip() for part in value.split(",") if part.strip()]
+
+        if isinstance(loaded, list):
+            return normalize_unique_non_empty_strings(loaded)
+        raise EvretValidationError("relevant_docs must be a JSON list or comma-separated string")

evret/evaluation/evaluator.py

@@ -0,0 +1,62 @@
+"""Evaluation orchestrator for retriever metrics."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from evret.errors import EvretValidationError
+from evret.evaluation.dataset import EvaluationDataset
+from evret.evaluation.results import EvaluationResults
+from evret.metrics import Metric
+from evret.retrievers import BaseRetriever
+from evret.utils import find_duplicates
+
+
+class Evaluator:
+    """Run a list of metrics over a retriever and dataset."""
+
+    def __init__(self, retriever: BaseRetriever, metrics: Sequence[Metric]) -> None:
+        if not metrics:
+            raise EvretValidationError("metrics must contain at least one metric")
+
+        metric_names = [metric.name for metric in metrics]
+        duplicate_metric_names = sorted(find_duplicates(metric_names))
+        if duplicate_metric_names:
+            duplicates_text = ", ".join(duplicate_metric_names)
+            raise EvretValidationError(
+                f"duplicate metric names are not allowed: {duplicates_text}"
+            )
+
+        self.retriever = retriever
+        self.metrics = list(metrics)
+
+    def evaluate(self, dataset: EvaluationDataset) -> EvaluationResults:
+        if not dataset.queries:
+            raise EvretValidationError("dataset must contain at least one query")
+
+        max_k = max(metric.k for metric in self.metrics)
+        retrieved_results = self.retriever.batch_retrieve(
+            queries=[query.query_text for query in dataset.queries],
+            k=max_k,
+        )
+        if len(retrieved_results) != len(dataset.queries):
+            raise EvretValidationError(
+                "retriever returned a different number of query results than input queries"
+            )
+        retrieved_ids = [
+            [result.doc_id for result in query_results]
+            for query_results in retrieved_results
+        ]
+        relevant_sets = [set(query.relevant_doc_ids) for query in dataset.queries]
+
+        metric_scores: dict[str, float] = {}
+        for metric in self.metrics:
+            metric_scores[metric.name] = metric.score(
+                retrieved_by_query=retrieved_ids,
+                relevant_by_query=relevant_sets,
+            )
+
+        return EvaluationResults(
+            metric_scores=metric_scores,
+            query_count=len(dataset.queries),
+        )

evret/evaluation/results.py

@@ -0,0 +1,63 @@
+"""Evaluation result container and exporters."""
+
+from __future__ import annotations
+
+import csv
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from evret.errors import EvretValidationError
+from evret.utils import ensure_parent_dir, require_non_empty_str
+
+
+@dataclass(slots=True)
+class EvaluationResults:
+    """Aggregated metric results for an evaluation run."""
+
+    metric_scores: dict[str, float]
+    query_count: int
+    generated_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+    def __post_init__(self) -> None:
+        if self.query_count < 0:
+            raise EvretValidationError("query_count must be non-negative")
+
+        normalized_scores: dict[str, float] = {}
+        for metric_name, score in self.metric_scores.items():
+            normalized_metric_name = require_non_empty_str(metric_name, "metric name")
+            normalized_scores[normalized_metric_name] = float(score)
+        self.metric_scores = normalized_scores
+
+    def summary(self) -> dict[str, float]:
+        """Return metric summary map."""
+        return dict(self.metric_scores)
+
+    def to_dict(self) -> dict[str, object]:
+        """Return serializable representation of this result object."""
+        return {
+            "query_count": self.query_count,
+            "generated_at": self.generated_at,
+            "metrics": self.summary(),
+        }
+
+    def to_json(self, path: str | Path) -> None:
+        """Write results as JSON."""
+        output_path = ensure_parent_dir(path)
+        output_path.write_text(
+            json.dumps(self.to_dict(), indent=2, sort_keys=True),
+            encoding="utf-8",
+        )
+
+    def to_csv(self, path: str | Path) -> None:
+        """Write metric rows as CSV (`metric`, `score`)."""
+        output_path = ensure_parent_dir(path)
+        with output_path.open("w", encoding="utf-8", newline="") as handle:
+            writer = csv.writer(handle)
+            writer.writerow(["metric", "score"])
+            for metric_name in sorted(self.metric_scores):
+                score = self.metric_scores[metric_name]
+                writer.writerow([metric_name, f"{score:.12g}"])

evret/integrations/__init__.py

@@ -0,0 +1,6 @@
+"""Integration exports."""
+
+from evret.integrations.langchain import LangChainRetrieverAdapter
+from evret.integrations.llamaindex import LlamaIndexRetrieverAdapter
+
+__all__ = ["LangChainRetrieverAdapter", "LlamaIndexRetrieverAdapter"]

evret/integrations/langchain.py

@@ -0,0 +1,66 @@
+"""LangChain adapter for Evret retrievers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from evret.errors import OptionalDependencyError
+from evret.retrievers import BaseRetriever
+from evret.utils import require_non_empty_str, require_positive_int
+
+try:
+    from langchain_core.documents import Document
+    from langchain_core.retrievers import BaseRetriever as LangChainBaseRetriever
+    from pydantic import ConfigDict
+
+    _LANGCHAIN_AVAILABLE = True
+except ImportError:  # pragma: no cover - exercised when optional dependency is absent
+    Document = None
+    LangChainBaseRetriever = object  # type: ignore[assignment]
+    ConfigDict = None  # type: ignore[assignment]
+    _LANGCHAIN_AVAILABLE = False
+
+if TYPE_CHECKING:
+    from langchain_core.documents import Document as LangChainDocument
+
+
+class LangChainRetrieverAdapter(LangChainBaseRetriever):
+    """Wrap an Evret retriever as a LangChain-compatible retriever."""
+
+    if _LANGCHAIN_AVAILABLE:
+        model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    evret_retriever: BaseRetriever
+    k: int = 4
+
+    def __init__(self, evret_retriever: BaseRetriever, k: int = 4, **kwargs: Any) -> None:
+        if not _LANGCHAIN_AVAILABLE:
+            raise OptionalDependencyError(
+                "langchain-core is required for LangChainRetrieverAdapter. "
+                "Install with: pip install langchain-core"
+            )
+        super().__init__(evret_retriever=evret_retriever, k=self._coerce_k(k), **kwargs)
+
+    def _get_relevant_documents(self, query: str, **kwargs: Any) -> list[LangChainDocument]:
+        normalized_query = require_non_empty_str(query, "query")
+
+        k = self._coerce_k(kwargs.get("k", self.k))
+
+        results = self.evret_retriever.retrieve(query=normalized_query, k=k)
+        documents: list[Document] = []
+        for result in results:
+            metadata = dict(result.metadata)
+            page_content = str(metadata.pop("document", ""))
+            metadata.setdefault("doc_id", result.doc_id)
+            metadata.setdefault("score", result.score)
+            documents.append(Document(page_content=page_content, metadata=metadata))
+        return documents
+
+    async def _aget_relevant_documents(
+        self, query: str, **kwargs: Any
+    ) -> list[LangChainDocument]:
+        return self._get_relevant_documents(query=query, **kwargs)
+
+    @staticmethod
+    def _coerce_k(value: Any) -> int:
+        return require_positive_int(value, "k")

evret/integrations/llamaindex.py

@@ -0,0 +1,66 @@
+"""LlamaIndex adapter for Evret retrievers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from evret.errors import OptionalDependencyError
+from evret.retrievers import BaseRetriever
+from evret.utils import require_non_empty_str, require_positive_int
+
+try:
+    from llama_index.core import QueryBundle
+    from llama_index.core.retrievers import BaseRetriever as LlamaIndexBaseRetriever
+    from llama_index.core.schema import NodeWithScore, TextNode
+
+    _LLAMAINDEX_AVAILABLE = True
+except ImportError:  # pragma: no cover - exercised when optional dependency is absent
+    QueryBundle = None  # type: ignore[assignment]
+    LlamaIndexBaseRetriever = object  # type: ignore[assignment]
+    NodeWithScore = None  # type: ignore[assignment]
+    TextNode = None  # type: ignore[assignment]
+    _LLAMAINDEX_AVAILABLE = False
+
+if TYPE_CHECKING:
+    from llama_index.core import QueryBundle as LlamaIndexQueryBundle
+    from llama_index.core.schema import NodeWithScore as LlamaIndexNodeWithScore
+
+
+class LlamaIndexRetrieverAdapter(LlamaIndexBaseRetriever):
+    """Wrap an Evret retriever as a LlamaIndex-compatible retriever."""
+
+    def __init__(
+        self,
+        evret_retriever: BaseRetriever,
+        k: int = 4,
+        *,
+        text_field: str = "document",
+        **kwargs: Any,
+    ) -> None:
+        if not _LLAMAINDEX_AVAILABLE:
+            raise OptionalDependencyError(
+                "llama-index-core is required for LlamaIndexRetrieverAdapter. "
+                "Install with: pip install llama-index-core"
+            )
+
+        super().__init__(**kwargs)
+        self.evret_retriever = evret_retriever
+        self.k = require_positive_int(k, "k")
+        self.text_field = require_non_empty_str(text_field, "text_field")
+
+    def _retrieve(self, query_bundle: LlamaIndexQueryBundle) -> list[LlamaIndexNodeWithScore]:
+        query = require_non_empty_str(query_bundle.query_str, "query")
+        results = self.evret_retriever.retrieve(query=query, k=self.k)
+
+        nodes: list[NodeWithScore] = []
+        for result in results:
+            metadata = dict(result.metadata)
+            node_text = str(metadata.pop(self.text_field, ""))
+            metadata.setdefault("doc_id", result.doc_id)
+            metadata.setdefault("score", result.score)
+            node = TextNode(text=node_text, id_=result.doc_id, metadata=metadata)
+            nodes.append(NodeWithScore(node=node, score=result.score))
+        return nodes
+
+    async def _aretrieve(self, query_bundle: LlamaIndexQueryBundle) -> list[LlamaIndexNodeWithScore]:
+        return self._retrieve(query_bundle)

evret/metrics/__init__.py

@@ -0,0 +1,19 @@
+"""Metric exports."""
+
+from evret.metrics.average_precision import AveragePrecision
+from evret.metrics.base import Metric
+from evret.metrics.hit_rate import HitRate
+from evret.metrics.mrr import MRR
+from evret.metrics.ndcg import NDCG
+from evret.metrics.precision import Precision
+from evret.metrics.recall import Recall
+
+__all__ = [
+    "Metric",
+    "HitRate",
+    "Recall",
+    "Precision",
+    "MRR",
+    "NDCG",
+    "AveragePrecision",
+]

evret/metrics/average_precision.py

@@ -0,0 +1,35 @@
+"""Average Precision@K metric implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Sequence
+
+from evret.metrics.base import Metric
+
+
+class AveragePrecision(Metric):
+    """Average Precision at top-k cutoff."""
+
+    metric_name = "average_precision"
+
+    def score_query(
+        self,
+        retrieved_doc_ids: Sequence[str],
+        relevant_doc_ids: Collection[str],
+    ) -> float:
+        relevant_set = self._relevant_set(relevant_doc_ids)
+        if not relevant_set:
+            return 0.0
+
+        hit_count = 0
+        precision_sum = 0.0
+        for rank, doc_id in enumerate(self.top_k(retrieved_doc_ids), start=1):
+            if doc_id in relevant_set:
+                hit_count += 1
+                precision_sum += hit_count / rank
+
+        return precision_sum / len(relevant_set)
+
+    @staticmethod
+    def _relevant_set(relevant_doc_ids: Collection[str]) -> set[str]:
+        return set(relevant_doc_ids)

evret/metrics/base.py

@@ -0,0 +1,64 @@
+"""Base interface for retrieval evaluation metrics."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Collection, Sequence
+
+from evret.errors import EvretValidationError
+from evret.utils import require_positive_int
+
+
+class Metric(ABC):
+    """Base class for metrics evaluated at a top-k cutoff.
+
+    For query ``i`` with retrieved document IDs ``R_i`` and relevant IDs ``G_i``,
+    each metric computes a per-query score at ``k`` and then averages:
+
+    ``score = (1 / |Q|) * sum(metric_i(R_i[:k], G_i))``
+    """
+
+    metric_name: str = "metric"
+
+    def __init__(self, k: int) -> None:
+        self.k = require_positive_int(k, "k")
+
+    @property
+    def name(self) -> str:
+        """Metric display name including cutoff."""
+        return f"{self.metric_name}@{self.k}"
+
+    @abstractmethod
+    def score_query(
+        self,
+        retrieved_doc_ids: Sequence[str],
+        relevant_doc_ids: Collection[str],
+    ) -> float:
+        """Score a single query."""
+
+    def score(
+        self,
+        retrieved_by_query: Sequence[Sequence[str]],
+        relevant_by_query: Sequence[Collection[str]],
+    ) -> float:
+        """Score a batch of queries by averaging per-query metric values."""
+        if len(retrieved_by_query) != len(relevant_by_query):
+            raise EvretValidationError(
+                "retrieved_by_query and relevant_by_query must have same length"
+            )
+
+        if not retrieved_by_query:
+            return 0.0
+
+        total = 0.0
+        for retrieved, relevant in zip(retrieved_by_query, relevant_by_query):
+            total += self.score_query(
+                retrieved_doc_ids=retrieved,
+                relevant_doc_ids=relevant,
+            )
+
+        return total / len(retrieved_by_query)
+
+    def top_k(self, retrieved_doc_ids: Sequence[str]) -> Sequence[str]:
+        """Return the retrieval list trimmed to metric cutoff."""
+        return retrieved_doc_ids[: self.k]

evret/metrics/hit_rate.py

@@ -0,0 +1,26 @@
+"""Hit Rate metric implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Sequence
+
+from evret.metrics.base import Metric
+
+
+class HitRate(Metric):
+    """Binary top-k relevance presence metric.
+
+    Formula:
+    ``HitRate@k = (1 / |Q|) * sum(1[relevant_i ∩ retrieved_i[:k] != ∅])``
+    """
+
+    metric_name = "hit_rate"
+
+    def score_query(
+        self,
+        retrieved_doc_ids: Sequence[str],
+        relevant_doc_ids: Collection[str],
+    ) -> float:
+        top_k_ids = set(self.top_k(retrieved_doc_ids))
+        relevant_ids = set(relevant_doc_ids)
+        return 1.0 if top_k_ids.intersection(relevant_ids) else 0.0

evret/metrics/mrr.py

@@ -0,0 +1,31 @@
+"""MRR@K metric implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Sequence
+
+from evret.metrics.base import Metric
+
+
+class MRR(Metric):
+    """Mean Reciprocal Rank query metric at top-k.
+
+    Formula:
+    ``RR@k = 1 / rank_first_relevant`` if a hit exists in top-k, else ``0``.
+    """
+
+    metric_name = "mrr"
+
+    def score_query(
+        self,
+        retrieved_doc_ids: Sequence[str],
+        relevant_doc_ids: Collection[str],
+    ) -> float:
+        relevant_ids = set(relevant_doc_ids)
+        if not relevant_ids:
+            return 0.0
+
+        for rank, doc_id in enumerate(self.top_k(retrieved_doc_ids), start=1):
+            if doc_id in relevant_ids:
+                return 1.0 / rank
+        return 0.0