evret 0.0.1__py3-none-any.whl

evret/__init__.py

@@ -0,0 +1,57 @@
+"""Evret package."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from evret.evaluation import (
+    DocumentExample,
+    EvaluationDataset,
+    EvaluationResults,
+    Evaluator,
+    QueryExample,
+)
+from evret.errors import EvretError, EvretValidationError, OptionalDependencyError
+from evret.integrations import LangChainRetrieverAdapter, LlamaIndexRetrieverAdapter
+from evret.judges import Judge, JudgmentContext, TokenOverlapJudge
+from evret.metrics import AveragePrecision, HitRate, MRR, NDCG, Precision, Recall
+from evret.retrievers import (
+    BaseRetriever,
+    ChromaRetriever,
+    MilvusRetriever,
+    QdrantRetriever,
+    RetrievalResult,
+    WeaviateRetriever,
+)
+
+try:
+    __version__ = version("evret")
+except PackageNotFoundError:  # pragma: no cover - only when package metadata is unavailable
+    __version__ = "0.0.0"
+
+__all__ = [
+    "__version__",
+    "AveragePrecision",
+    "BaseRetriever",
+    "ChromaRetriever",
+    "DocumentExample",
+    "EvaluationDataset",
+    "EvaluationResults",
+    "Evaluator",
+    "EvretError",
+    "EvretValidationError",
+    "HitRate",
+    "Judge",
+    "JudgmentContext",
+    "LangChainRetrieverAdapter",
+    "LlamaIndexRetrieverAdapter",
+    "MilvusRetriever",
+    "MRR",
+    "NDCG",
+    "OptionalDependencyError",
+    "Precision",
+    "QueryExample",
+    "QdrantRetriever",
+    "Recall",
+    "RetrievalResult",
+    "TokenOverlapJudge",
+    "WeaviateRetriever",
+]

evret/errors.py

@@ -0,0 +1,15 @@
+"""Evret exception hierarchy."""
+
+from __future__ import annotations
+
+
+class EvretError(Exception):
+    """Base exception for Evret."""
+
+
+class EvretValidationError(ValueError, EvretError):
+    """Raised when user input or data format is invalid."""
+
+
+class OptionalDependencyError(ImportError, EvretError):
+    """Raised when an optional dependency required by a feature is missing."""

evret/evaluation/__init__.py

@@ -0,0 +1,13 @@
+"""Evaluation exports."""
+
+from evret.evaluation.dataset import DocumentExample, EvaluationDataset, QueryExample
+from evret.evaluation.evaluator import Evaluator
+from evret.evaluation.results import EvaluationResults
+
+__all__ = [
+    "QueryExample",
+    "DocumentExample",
+    "EvaluationDataset",
+    "Evaluator",
+    "EvaluationResults",
+]

evret/evaluation/dataset.py

@@ -0,0 +1,168 @@
+"""Evaluation dataset models and loaders."""
+
+from __future__ import annotations
+
+import csv
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from evret.errors import EvretValidationError
+from evret.utils import (
+    normalize_unique_non_empty_strings,
+    require_file_exists,
+    require_non_empty_str,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class QueryExample:
+    """One query item in an evaluation dataset.
+
+    Supports two evaluation patterns:
+    1. Classic IR: Provide relevant_doc_ids (pre-labeled document identifiers)
+    2. Judge-based: Provide expected_answers (answer text snippets for judge to match)
+
+    Use relevant_doc_ids when you have pre-labeled ground truth document IDs.
+    Use expected_answers when you want a judge to determine relevance by comparing against expected answer text.
+    """
+
+    query_id: str
+    query_text: str
+    relevant_doc_ids: list[str] = field(default_factory=list)
+    expected_answers: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True, slots=True)
+class DocumentExample:
+    """One document entry in an evaluation dataset."""
+
+    doc_id: str
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class EvaluationDataset:
+    """Evaluation dataset containing query examples and optional documents."""
+
+    queries: list[QueryExample]
+    documents: list[DocumentExample] = field(default_factory=list)
+
+    @classmethod
+    def from_json(cls, path: str | Path) -> EvaluationDataset:
+        dataset_path = require_file_exists(path, "dataset")
+        raw_data = json.loads(dataset_path.read_text(encoding="utf-8"))
+        if not isinstance(raw_data, dict):
+            raise EvretValidationError("root JSON value must be an object")
+
+        raw_queries = raw_data.get("queries", [])
+        raw_documents = raw_data.get("documents", [])
+        if not isinstance(raw_queries, list):
+            raise EvretValidationError("queries must be a list")
+        if not isinstance(raw_documents, list):
+            raise EvretValidationError("documents must be a list")
+
+        queries = [cls._parse_query_item(item) for item in raw_queries]
+        documents = [cls._parse_document_item(item) for item in raw_documents]
+        return cls(queries=queries, documents=documents)
+
+    @classmethod
+    def from_csv(cls, path: str | Path) -> EvaluationDataset:
+        dataset_path = require_file_exists(path, "dataset")
+        query_items: list[QueryExample] = []
+        with dataset_path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            if reader.fieldnames is None:
+                raise EvretValidationError("CSV must include a header row")
+            for row_index, row in enumerate(reader, start=1):
+                query_text = (row.get("query_text") or row.get("query") or "").strip()
+                if not query_text:
+                    raise EvretValidationError(f"query_text is required in row {row_index}")
+
+                query_id = (
+                    row.get("query_id")
+                    or row.get("id")
+                    or f"q{row_index}"
+                )
+                query_id = require_non_empty_str(query_id, "query_id")
+
+                relevant_doc_ids = cls._parse_relevant_docs(row.get("relevant_doc_ids", ""))
+                expected_answers = cls._parse_relevant_docs(row.get("expected_answers", ""))
+
+                if not relevant_doc_ids and not expected_answers:
+                    relevant_doc_ids = cls._parse_relevant_docs(row.get("relevant_docs", ""))
+
+                query_items.append(
+                    QueryExample(
+                        query_id=query_id,
+                        query_text=query_text,
+                        relevant_doc_ids=relevant_doc_ids,
+                        expected_answers=expected_answers,
+                    )
+                )
+
+        return cls(queries=query_items, documents=[])
+
+    @staticmethod
+    def _parse_query_item(item: Any) -> QueryExample:
+        if not isinstance(item, dict):
+            raise EvretValidationError("each query item must be an object")
+
+        query_id_raw = item.get("query_id") or item.get("id")
+        query_text_raw = item.get("query_text") or item.get("query")
+
+        query_id = require_non_empty_str(query_id_raw or "", "query_id")
+        query_text = require_non_empty_str(query_text_raw or "", "query_text")
+
+        relevant_doc_ids_raw = item.get("relevant_doc_ids") or []
+        if not isinstance(relevant_doc_ids_raw, list):
+            raise EvretValidationError("relevant_doc_ids must be a list")
+        relevant_doc_ids = normalize_unique_non_empty_strings(relevant_doc_ids_raw)
+
+        expected_answers_raw = item.get("expected_answers") or []
+        if not isinstance(expected_answers_raw, list):
+            raise EvretValidationError("expected_answers must be a list")
+        expected_answers = normalize_unique_non_empty_strings(expected_answers_raw)
+
+        if not relevant_doc_ids and not expected_answers:
+            relevant_docs_raw = item.get("relevant_docs") or []
+            if not isinstance(relevant_docs_raw, list):
+                raise EvretValidationError("relevant_docs must be a list")
+            relevant_doc_ids = normalize_unique_non_empty_strings(relevant_docs_raw)
+
+        return QueryExample(
+            query_id=query_id,
+            query_text=query_text,
+            relevant_doc_ids=relevant_doc_ids,
+            expected_answers=expected_answers,
+        )
+
+    @staticmethod
+    def _parse_document_item(item: Any) -> DocumentExample:
+        if not isinstance(item, dict):
+            raise EvretValidationError("each document item must be an object")
+
+        doc_id = require_non_empty_str(item.get("doc_id") or "", "doc_id")
+        text = str(item.get("text") or "")
+        metadata = item.get("metadata") or {}
+        if not isinstance(metadata, dict):
+            raise EvretValidationError("document metadata must be an object")
+
+        return DocumentExample(doc_id=doc_id, text=text, metadata=dict(metadata))
+
+    @staticmethod
+    def _parse_relevant_docs(raw_value: str) -> list[str]:
+        value = raw_value.strip()
+        if not value:
+            return []
+
+        try:
+            loaded = json.loads(value)
+        except json.JSONDecodeError:
+            loaded = [part.strip() for part in value.split(",") if part.strip()]
+
+        if isinstance(loaded, list):
+            return normalize_unique_non_empty_strings(loaded)
+        raise EvretValidationError("relevant_docs must be a JSON list or comma-separated string")

evret/evaluation/evaluator.py

@@ -0,0 +1,224 @@
+"""Evaluation orchestrator for retriever metrics."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from evret.errors import EvretValidationError
+from evret.evaluation.dataset import EvaluationDataset
+from evret.evaluation.results import EvaluationResults
+from evret.judges.base import Judge, JudgmentContext
+from evret.judges.token_overlap import TokenOverlapJudge
+from evret.metrics import Metric
+from evret.retrievers import BaseRetriever, RetrievalResult
+from evret.utils import find_duplicates
+
+
+class Evaluator:
+    """Run a list of metrics over a retriever and dataset.
+
+    Uses pluggable Judge system for text-based relevance matching.
+
+    Args:
+        retriever: Retriever to evaluate
+        metrics: List of metrics to compute
+        judge: Relevance judge (defaults to TokenOverlapJudge if None)
+
+    Examples:
+        >>> from evret import Evaluator, HitRate, Recall
+        >>> from evret.judges import TokenOverlapJudge, SemanticJudge, LLMJudge
+        >>>
+        >>> # Default: TokenOverlapJudge
+        >>> evaluator = Evaluator(retriever, [HitRate(k=4), Recall(k=4)])
+        >>>
+        >>> # Custom judge
+        >>> evaluator = Evaluator(
+        ...     retriever,
+        ...     [Recall(k=4)],
+        ...     judge=SemanticJudge(threshold=0.8)
+        ... )
+    """
+
+    def __init__(
+        self,
+        retriever: BaseRetriever,
+        metrics: Sequence[Metric],
+        judge: Judge | None = None,
+    ) -> None:
+        if not metrics:
+            raise EvretValidationError("metrics must contain at least one metric")
+
+        metric_names = [metric.name for metric in metrics]
+        duplicate_metric_names = sorted(find_duplicates(metric_names))
+        if duplicate_metric_names:
+            duplicates_text = ", ".join(duplicate_metric_names)
+            raise EvretValidationError(
+                f"duplicate metric names are not allowed: {duplicates_text}"
+            )
+
+        self.retriever = retriever
+        self.metrics = list(metrics)
+        self.judge = judge or TokenOverlapJudge()
+
+    def evaluate(self, dataset: EvaluationDataset) -> EvaluationResults:
+        if not dataset.queries:
+            raise EvretValidationError("dataset must contain at least one query")
+
+        max_k = max(metric.k for metric in self.metrics)
+        retrieved_results = self.retriever.batch_retrieve(
+            queries=[query.query_text for query in dataset.queries],
+            k=max_k,
+        )
+        if len(retrieved_results) != len(dataset.queries):
+            raise EvretValidationError(
+                "retriever returned a different number of query results than input queries"
+            )
+        retrieved_ids, relevant_sets = self._build_metric_inputs(
+            dataset=dataset,
+            retrieved_results=retrieved_results,
+        )
+
+        metric_scores: dict[str, float] = {}
+        for metric in self.metrics:
+            metric_scores[metric.name] = metric.score(
+                retrieved_by_query=retrieved_ids,
+                relevant_by_query=relevant_sets,
+            )
+
+        return EvaluationResults(
+            metric_scores=metric_scores,
+            query_count=len(dataset.queries),
+        )
+
+    def _build_metric_inputs(
+        self,
+        dataset: EvaluationDataset,
+        retrieved_results: Sequence[Sequence[RetrievalResult]],
+    ) -> tuple[list[list[str]], list[set[str]]]:
+        """Build metric inputs using Judge system for text-based matching."""
+        document_text_by_id = {
+            self._normalize_label(document.doc_id): self._normalize_label(document.text)
+            for document in dataset.documents
+            if document.text.strip()
+        }
+
+        retrieved_by_query: list[list[str]] = []
+        relevant_by_query: list[set[str]] = []
+
+        for query, query_results in zip(dataset.queries, retrieved_results):
+            normalized_query_text = self._normalize_label(query.query_text)
+            labels_to_match = query.relevant_doc_ids if query.relevant_doc_ids else query.expected_answers
+            relevant_labels = [
+                self._normalize_label(label)
+                for label in labels_to_match
+                if str(label).strip()
+            ]
+            unique_relevant_labels = list(dict.fromkeys(relevant_labels))
+            relevant_set = set(unique_relevant_labels)
+
+            contexts_per_result = []
+            candidates_per_result = []
+
+            for rank, result in enumerate(query_results):
+                candidates = self._candidate_labels(result, document_text_by_id)
+                candidates_per_result.append(candidates)
+
+                result_contexts = []
+                for label in unique_relevant_labels:
+                    for candidate in candidates:
+                        result_contexts.append(
+                            JudgmentContext(
+                                query=normalized_query_text,
+                                expected_text=label,
+                                retrieved_text=candidate,
+                            )
+                        )
+                contexts_per_result.append(result_contexts)
+
+            all_contexts = [ctx for contexts in contexts_per_result for ctx in contexts]
+            if all_contexts:
+                all_judgments = self.judge.batch_judge(all_contexts)
+            else:
+                all_judgments = []
+
+            judgment_idx = 0
+            available_labels = set(unique_relevant_labels)
+            normalized_retrieved: list[str] = []
+
+            for rank, candidates in enumerate(candidates_per_result):
+                num_contexts = len(unique_relevant_labels) * len(candidates)
+                result_judgments = all_judgments[judgment_idx:judgment_idx + num_contexts]
+                judgment_idx += num_contexts
+
+                matched_label = self._find_match(
+                    judgments=result_judgments,
+                    ordered_relevant_labels=unique_relevant_labels,
+                    candidates=candidates,
+                    remaining_relevant_labels=available_labels,
+                )
+
+                if matched_label is not None:
+                    normalized_retrieved.append(matched_label)
+                    available_labels.discard(matched_label)
+                else:
+                    fallback_label = candidates[0] if candidates else f"retrieved_{rank}"
+                    normalized_retrieved.append(f"retrieved_{rank}:{fallback_label}")
+
+            retrieved_by_query.append(normalized_retrieved)
+            relevant_by_query.append(relevant_set)
+
+        return retrieved_by_query, relevant_by_query
+
+    def _candidate_labels(
+        self,
+        result: RetrievalResult,
+        document_text_by_id: dict[str, str],
+    ) -> list[str]:
+        """Extract candidate text labels from retrieval result."""
+        candidate_labels: list[str] = [self._normalize_label(result.doc_id)]
+        doc_text = document_text_by_id.get(candidate_labels[0], "")
+        if doc_text:
+            candidate_labels.append(doc_text)
+
+        text_keys = (
+            "text",
+            "document",
+            "content",
+            "page_content",
+            "chunk",
+            "passage",
+        )
+        for key in text_keys:
+            value = result.metadata.get(key)
+            if value is None:
+                continue
+            normalized_value = self._normalize_label(value)
+            if normalized_value:
+                candidate_labels.append(normalized_value)
+
+        return list(dict.fromkeys(candidate_labels))
+
+    def _find_match(
+        self,
+        judgments: list[bool],
+        ordered_relevant_labels: Sequence[str],
+        candidates: list[str],
+        remaining_relevant_labels: set[str],
+    ) -> str | None:
+        """Find first matching label from batch judgments."""
+        idx = 0
+        for label in ordered_relevant_labels:
+            if label not in remaining_relevant_labels:
+                idx += len(candidates)
+                continue
+
+            for _ in candidates:
+                if judgments[idx]:
+                    return label
+                idx += 1
+
+        return None
+
+    @staticmethod
+    def _normalize_label(value: object) -> str:
+        return " ".join(str(value).strip().lower().split())

evret/evaluation/judges.py

@@ -0,0 +1,86 @@
+"""Relevance judge helpers for text-based evaluation matching."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from evret.errors import EvretValidationError
+RelevanceJudge = Callable[[str, str, str], bool]
+
+_TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
+
+
+def default_relevance_judge(query_text: str, relevant_label: str, candidate_text: str) -> bool:
+    """Return whether one candidate matches a relevance label."""
+    if relevant_label == candidate_text:
+        return True
+    if " " in relevant_label or " " in candidate_text:
+        if relevant_label in candidate_text or candidate_text in relevant_label:
+            return True
+    return token_overlap_relevance_judge(
+        query_text=query_text,
+        relevant_label=relevant_label,
+        candidate_text=candidate_text,
+    )
+
+
+def token_overlap_relevance_judge(
+    query_text: str,
+    relevant_label: str,
+    candidate_text: str,
+    *,
+    min_shared_tokens: int = 2,
+    min_overlap_ratio: float = 0.6,
+) -> bool:
+    """Return True when token overlap passes configured thresholds."""
+    if min_shared_tokens < 1:
+        raise EvretValidationError("min_shared_tokens must be >= 1")
+    if min_overlap_ratio <= 0 or min_overlap_ratio > 1:
+        raise EvretValidationError("min_overlap_ratio must be in (0, 1]")
+
+    label_tokens = set(_tokenize(relevant_label))
+    candidate_tokens = set(_tokenize(candidate_text))
+    if not label_tokens or not candidate_tokens:
+        return False
+
+    shared_tokens = label_tokens.intersection(candidate_tokens)
+    if len(shared_tokens) < min_shared_tokens:
+        return False
+
+    overlap_ratio = len(shared_tokens) / len(label_tokens)
+    if overlap_ratio >= min_overlap_ratio:
+        return True
+
+    query_tokens = set(_tokenize(query_text))
+    if not query_tokens:
+        return False
+
+    query_condition = len(shared_tokens.intersection(query_tokens)) >= 1
+    return query_condition and overlap_ratio >= (min_overlap_ratio * 0.75)
+
+
+def make_token_overlap_judge(
+    *,
+    min_shared_tokens: int = 2,
+    min_overlap_ratio: float = 0.6,
+) -> RelevanceJudge:
+    """Build a token-overlap relevance judge with custom thresholds."""
+    if min_shared_tokens < 1:
+        raise EvretValidationError("min_shared_tokens must be >= 1")
+    if min_overlap_ratio <= 0 or min_overlap_ratio > 1:
+        raise EvretValidationError("min_overlap_ratio must be in (0, 1]")
+
+    def judge(query_text: str, relevant_label: str, candidate_text: str) -> bool:
+        return token_overlap_relevance_judge(
+            query_text=query_text,
+            relevant_label=relevant_label,
+            candidate_text=candidate_text,
+            min_shared_tokens=min_shared_tokens,
+            min_overlap_ratio=min_overlap_ratio,
+        )
+
+    return judge
+
+
+def _tokenize(value: str) -> list[str]:
+    return _TOKEN_PATTERN.findall(value.lower())

evret/evaluation/results.py

@@ -0,0 +1,63 @@
+"""Evaluation result container and exporters."""
+
+from __future__ import annotations
+
+import csv
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from evret.errors import EvretValidationError
+from evret.utils import ensure_parent_dir, require_non_empty_str
+
+
+@dataclass(slots=True)
+class EvaluationResults:
+    """Aggregated metric results for an evaluation run."""
+
+    metric_scores: dict[str, float]
+    query_count: int
+    generated_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+    def __post_init__(self) -> None:
+        if self.query_count < 0:
+            raise EvretValidationError("query_count must be non-negative")
+
+        normalized_scores: dict[str, float] = {}
+        for metric_name, score in self.metric_scores.items():
+            normalized_metric_name = require_non_empty_str(metric_name, "metric name")
+            normalized_scores[normalized_metric_name] = float(score)
+        self.metric_scores = normalized_scores
+
+    def summary(self) -> dict[str, float]:
+        """Return metric summary map."""
+        return dict(self.metric_scores)
+
+    def to_dict(self) -> dict[str, object]:
+        """Return serializable representation of this result object."""
+        return {
+            "query_count": self.query_count,
+            "generated_at": self.generated_at,
+            "metrics": self.summary(),
+        }
+
+    def to_json(self, path: str | Path) -> None:
+        """Write results as JSON."""
+        output_path = ensure_parent_dir(path)
+        output_path.write_text(
+            json.dumps(self.to_dict(), indent=2, sort_keys=True),
+            encoding="utf-8",
+        )
+
+    def to_csv(self, path: str | Path) -> None:
+        """Write metric rows as CSV (`metric`, `score`)."""
+        output_path = ensure_parent_dir(path)
+        with output_path.open("w", encoding="utf-8", newline="") as handle:
+            writer = csv.writer(handle)
+            writer.writerow(["metric", "score"])
+            for metric_name in sorted(self.metric_scores):
+                score = self.metric_scores[metric_name]
+                writer.writerow([metric_name, f"{score:.12g}"])

evret/integrations/__init__.py

@@ -0,0 +1,6 @@
+"""Integration exports."""
+
+from evret.integrations.langchain import LangChainRetrieverAdapter
+from evret.integrations.llamaindex import LlamaIndexRetrieverAdapter
+
+__all__ = ["LangChainRetrieverAdapter", "LlamaIndexRetrieverAdapter"]