ragrefine 0.2.1__py3-none-any.whl

ragrefine/__init__.py

@@ -0,0 +1,14 @@
+"""Retriever-agnostic post-retrieval refinement for RAG candidate sets."""
+
+from ragrefine.config import RefinerConfig
+from ragrefine.models import Candidate, CandidateSet
+from ragrefine.refiner import Refiner
+from ragrefine.results import RefinementResult
+
+__all__ = [
+    "Candidate",
+    "CandidateSet",
+    "RefinerConfig",
+    "RefinementResult",
+    "Refiner",
+]

ragrefine/_channels.py

@@ -0,0 +1,16 @@
+"""Private channel execution result used only by :mod:`ragrefine.refiner`."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+
+from ragrefine.models import Candidate
+
+
+@dataclass(frozen=True, slots=True)
+class _ChannelResult:
+    """One successful channel execution over the unchanged input candidate pool."""
+
+    name: str
+    ranking: tuple[Candidate, ...]
+    evidence: Mapping[str, Mapping[str, object]]
+    metadata: Mapping[str, object] = field(default_factory=dict)

ragrefine/config.py

@@ -0,0 +1,27 @@
+"""Public configuration for deterministic refinement channels."""
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, slots=True)
+class ChannelConfig:
+    """Whether to run one channel, require success, and its fusion weight."""
+
+    enabled: bool = False
+    required: bool = True
+    weight: float = 1.0
+
+    def __post_init__(self) -> None:
+        if self.weight < 0:
+            raise ValueError("channel weight must be non-negative")
+
+
+@dataclass(frozen=True, slots=True)
+class RefinerConfig:
+    """Explicit independent channel configuration for B0 through B3 modes."""
+
+    original: ChannelConfig = field(default_factory=lambda: ChannelConfig(enabled=True))
+    neural: ChannelConfig = field(default_factory=ChannelConfig)
+    lexical: ChannelConfig = field(default_factory=ChannelConfig)
+    pattern: ChannelConfig = field(default_factory=ChannelConfig)
+    fusion_enabled: bool = False

ragrefine/errors.py

@@ -0,0 +1,21 @@
+"""Typed package exceptions for refinement failures."""
+
+
+class RagRefineError(Exception):
+    """Base exception for explicit ragrefine failures."""
+
+
+class RerankerError(RagRefineError):
+    """Raised when neural reranking cannot complete safely."""
+
+
+class PatternError(RagRefineError):
+    """Raised when configured pattern processing is invalid or unsafe."""
+
+
+class ChannelExecutionError(RagRefineError):
+    """Raised when a required ranking channel cannot complete."""
+
+
+class MultipleActiveChannelsError(RagRefineError):
+    """Raised when independent rankings need fusion but it is disabled."""

ragrefine/integrations/__init__.py

@@ -0,0 +1 @@
+"""Optional framework integrations for ragrefine."""

ragrefine/integrations/haystack/__init__.py

@@ -0,0 +1,33 @@
+"""Optional Haystack integration for ragrefine.
+
+Installed with ``ragrefine[haystack]``. Provides the document adapter,
+``RagRefineComponent`` with serializable refiner profiles, the prompt builder,
+a frozen retrieval fixture, and a ready-made RAG pipeline builder.
+"""
+
+from ragrefine.integrations.haystack.adapter import HaystackDocumentAdapter
+from ragrefine.integrations.haystack.component import (
+    RagRefineComponent,
+    refiner_for_profile,
+    register_refiner_factory,
+)
+from ragrefine.integrations.haystack.fixture import FrozenHaystackFixture
+from ragrefine.integrations.haystack.pipeline import (
+    ChatPromptAdapter,
+    build_rag_pipeline,
+    demo_documents,
+)
+from ragrefine.integrations.haystack.prompt import PROMPT_TEMPLATE, prompt_builder
+
+__all__ = [
+    "ChatPromptAdapter",
+    "FrozenHaystackFixture",
+    "HaystackDocumentAdapter",
+    "PROMPT_TEMPLATE",
+    "RagRefineComponent",
+    "build_rag_pipeline",
+    "demo_documents",
+    "prompt_builder",
+    "refiner_for_profile",
+    "register_refiner_factory",
+]

ragrefine/integrations/haystack/adapter.py

@@ -0,0 +1,72 @@
+"""Optional Haystack ``Document`` conversion without core-package coupling."""
+
+from collections.abc import Iterable, Sequence
+
+from haystack import Document
+
+from ragrefine.models import Candidate, CandidateSet, RefinedCandidate
+
+
+class HaystackDocumentAdapter:
+    """Adapt one ordered Haystack result set while retaining document identity.
+
+    The adapter owns no ranking logic. It converts documents to immutable core
+    candidates, then maps selected/refined candidates back to the exact original
+    ``Document`` objects supplied at construction.
+    """
+
+    def __init__(
+        self, documents: Sequence[Document], *, name: str = "haystack"
+    ) -> None:
+        if not name:
+            raise ValueError("candidate-set name must be non-empty")
+        self._documents = tuple(documents)
+        self._by_id: dict[str, Document] = {}
+        for document in self._documents:
+            if not isinstance(document, Document):
+                raise TypeError("documents must contain Haystack Document instances")
+            document_id = str(document.id)
+            if not document_id:
+                raise ValueError("Haystack Document IDs must be non-empty")
+            if document_id in self._by_id:
+                raise ValueError(f"duplicate Haystack Document ID: {document_id}")
+            self._by_id[document_id] = document
+        self._name = name
+
+    def candidate_set(self) -> CandidateSet:
+        """Return the ordered, provenance-preserving core candidate set."""
+        return CandidateSet(
+            name=self._name,
+            candidates=tuple(
+                Candidate(
+                    id=str(document.id),
+                    text=document.content or "",
+                    metadata=dict(document.meta),
+                    retrieval_score=document.score,
+                    retrieval_rank=index,
+                )
+                for index, document in enumerate(self._documents, start=1)
+            ),
+        )
+
+    def documents_for(
+        self, candidates: Iterable[RefinedCandidate | Candidate]
+    ) -> tuple[Document, ...]:
+        """Return the original documents for refined/selected core candidates."""
+        resolved: list[Document] = []
+        for item in candidates:
+            candidate = item.candidate if isinstance(item, RefinedCandidate) else item
+            if not isinstance(candidate, Candidate):
+                raise TypeError("candidates must contain Candidate or RefinedCandidate")
+            try:
+                document = self._by_id[candidate.id]
+            except KeyError as error:
+                raise ValueError(
+                    f"candidate {candidate.id!r} was not created by this adapter"
+                ) from error
+            if document.content != candidate.text:
+                raise ValueError(
+                    f"candidate {candidate.id!r} text does not match source"
+                )
+            resolved.append(document)
+        return tuple(resolved)

ragrefine/integrations/haystack/component.py

@@ -0,0 +1,150 @@
+"""Native optional Haystack component for post-retrieval refinement."""
+
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+from ragrefine import Refiner, RefinerConfig
+from ragrefine.config import ChannelConfig
+from ragrefine.integrations.haystack.adapter import HaystackDocumentAdapter
+from ragrefine.ranking.lexical import LexicalRanker
+
+_REFINER_FACTORIES: dict[str, Callable[[], Refiner]] = {}
+
+
+def register_refiner_factory(name: str, factory: Callable[[], Refiner]) -> None:
+    """Register a named refiner profile for serializable components.
+
+    Built-in profiles are ``b0`` (original retrieval order) and
+    ``b2-l-lexical`` (the retained lightweight lexical profile). A custom
+    profile must be registered before a component using it is serialized, so
+    Haystack pipeline YAML can rebuild the exact refiner on load.
+    """
+    if not isinstance(name, str) or not name:
+        raise ValueError("refiner profile name must be a non-empty string")
+    if not callable(factory):
+        raise TypeError("refiner factory must be callable")
+    if name in _REFINER_FACTORIES:
+        raise ValueError(f"refiner profile already registered: {name!r}")
+    _REFINER_FACTORIES[name] = factory
+
+
+def refiner_for_profile(name: str) -> Refiner:
+    """Build the refiner for one registered profile name."""
+    try:
+        return _REFINER_FACTORIES[name]()
+    except KeyError as error:
+        raise ValueError(
+            f"unknown refiner profile {name!r}; "
+            f"registered: {sorted(_REFINER_FACTORIES)}"
+        ) from error
+
+
+def _b0_refiner() -> Refiner:
+    """Original-order (B0) profile: refinement leaves the pool order unchanged."""
+    return Refiner()
+
+
+def _b2_l_lexical_refiner() -> Refiner:
+    """Retained B2-L profile: stdlib lexical-coverage ranking only."""
+    return Refiner(
+        lexical_ranker=LexicalRanker(),
+        config=RefinerConfig(
+            original=ChannelConfig(enabled=False),
+            lexical=ChannelConfig(enabled=True),
+        ),
+    )
+
+
+register_refiner_factory("b0", _b0_refiner)
+register_refiner_factory("b2-l-lexical", _b2_l_lexical_refiner)
+
+
+@component
+class RagRefineComponent:
+    """Run a configured core refiner over one Haystack document result set.
+
+    The component intentionally delegates all ranking and selection semantics to
+    ``Refiner``. It returns the original document objects in the selected final
+    order plus a JSON-compatible trace. Built-in serializable profiles are
+    ``b0`` and ``b2-l-lexical``; custom refiners need a registered factory.
+    """
+
+    def __init__(
+        self,
+        *,
+        refiner: Refiner | None = None,
+        candidate_set_name: str = "haystack",
+        top_k: int = 5,
+        max_tokens: int | None = None,
+    ) -> None:
+        if top_k < 0:
+            raise ValueError("top_k must be non-negative")
+        if not candidate_set_name:
+            raise ValueError("candidate_set_name must be non-empty")
+        self._refiner = refiner or Refiner()
+        self._candidate_set_name = candidate_set_name
+        self._top_k = top_k
+        self._max_tokens = max_tokens
+
+    @component.output_types(documents=list[Document], trace=dict[str, Any])
+    def run(self, query: str, documents: list[Document]) -> dict[str, object]:
+        """Refine retrieved documents, preserving final core ordering exactly."""
+        if not isinstance(query, str):
+            raise TypeError("query must be a string")
+        adapter = HaystackDocumentAdapter(documents, name=self._candidate_set_name)
+        result = self._refiner.refine(
+            query,
+            adapter.candidate_set(),
+            top_k=self._top_k,
+            max_tokens=self._max_tokens,
+        )
+        return {
+            "documents": list(adapter.documents_for(result.candidates)),
+            "trace": asdict(result.trace),
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component with its named refiner profile."""
+        return default_to_dict(
+            self,
+            candidate_set_name=self._candidate_set_name,
+            top_k=self._top_k,
+            max_tokens=self._max_tokens,
+            refiner_profile=self._profile_name(),
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "RagRefineComponent":
+        """Recreate a component from its serialized profile and settings."""
+        init_parameters = dict(data.get("init_parameters", {}))
+        profile = init_parameters.pop("refiner_profile", "b0")
+        return cls(refiner=refiner_for_profile(profile), **init_parameters)
+
+    def _profile_name(self) -> str:
+        for name, factory in _REFINER_FACTORIES.items():
+            try:
+                candidate = factory()
+            except Exception:
+                continue
+            if self._equivalent(candidate):
+                return name
+        raise ValueError(
+            "serialization requires a registered refiner profile; register one "
+            "with register_refiner_factory(...) "
+            f"(registered: {sorted(_REFINER_FACTORIES)})"
+        )
+
+    def _equivalent(self, other: "Refiner") -> bool:
+        # SLF001 - serialization boundary; compares the exact ranking wiring.
+        return (
+            self._refiner._config == other._config
+            and type(self._refiner._reranker) is type(other._reranker)
+            and type(self._refiner._lexical_ranker) is type(other._lexical_ranker)
+            and type(self._refiner._pattern_ranker) is type(other._pattern_ranker)
+            and type(self._refiner._fusion) is type(other._fusion)
+            and type(self._refiner._deduplicator) is type(other._deduplicator)
+            and type(self._refiner._selector) is type(other._selector)
+        )

ragrefine/integrations/haystack/fixture.py

@@ -0,0 +1,96 @@
+"""Frozen, reproducible Haystack retrieval fixture independent of ragrefine."""
+
+import hashlib
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, cast
+
+from haystack import Document
+from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+
+FROZEN_CORPUS = (
+    {
+        "id": "python-311",
+        "content": "Python 3.11 adds exception groups.",
+        "meta": {"topic": "python"},
+    },
+    {
+        "id": "python-312",
+        "content": "Python 3.12 improves the f-string parser.",
+        "meta": {"topic": "python"},
+    },
+    {
+        "id": "haystack",
+        "content": "Haystack pipelines connect document retrieval components.",
+        "meta": {"topic": "rag"},
+    },
+)
+FROZEN_QUERIES = {
+    "python": "Which Python version improves f-string parsing?",
+    "rag": "How do Haystack pipelines retrieve documents?",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class RetrievalIdentity:
+    """Persistable identity for the frozen retrieval corpus and configuration."""
+
+    corpus_sha256: str
+    queries_sha256: str
+    retriever: str
+    document_store: str
+    top_n: int
+
+
+class FrozenHaystackFixture:
+    """Build one real deterministic BM25 document-store retrieval fixture."""
+
+    def __init__(self, *, top_n: int = 2) -> None:
+        if top_n < 1:
+            raise ValueError("top_n must be positive")
+        self._top_n = top_n
+        self._store = InMemoryDocumentStore(bm25_algorithm="BM25L")
+        self._store.write_documents(
+            [
+                Document(
+                    id=str(row["id"]),
+                    content=str(row["content"]),
+                    meta=dict(cast(Mapping[str, Any], row["meta"])),
+                )
+                for row in FROZEN_CORPUS
+            ]
+        )
+        self._retriever = InMemoryBM25Retriever(document_store=self._store)
+
+    def identity(self) -> RetrievalIdentity:
+        """Return frozen corpus/query/retriever/index identity for persistence."""
+        return RetrievalIdentity(
+            corpus_sha256=_checksum(FROZEN_CORPUS),
+            queries_sha256=_checksum(FROZEN_QUERIES),
+            retriever="haystack.InMemoryBM25Retriever/BM25L",
+            document_store="haystack.InMemoryDocumentStore",
+            top_n=self._top_n,
+        )
+
+    def retrieve(self, query: str) -> tuple[Document, ...]:
+        """Return the single frozen pool to reuse for every comparison branch."""
+        if not isinstance(query, str) or not query:
+            raise ValueError("query must be a non-empty string")
+        documents = self._retriever.run(query=query, top_k=self._top_n)["documents"]
+        if any(document.score is None for document in documents):
+            raise RuntimeError("Haystack retriever returned a document without a score")
+        return tuple(documents)
+
+    def frozen_query(self, name: str) -> str:
+        """Return one named immutable fixture query."""
+        try:
+            return FROZEN_QUERIES[name]
+        except KeyError as error:
+            raise ValueError(f"unknown frozen query: {name}") from error
+
+
+def _checksum(value: object) -> str:
+    encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
+    return hashlib.sha256(encoded).hexdigest()

ragrefine/integrations/haystack/pipeline.py

@@ -0,0 +1,104 @@
+"""Runnable Haystack RAG pipeline wiring for the ragrefine integration.
+
+``build_rag_pipeline`` wires the frozen BM25 retriever, ``RagRefineComponent``
+with a registered refiner profile, the fixed prompt builder, and an optional
+chat generator:
+
+```text
+InMemoryBM25Retriever -> RagRefineComponent -> PromptBuilder -> ChatPromptAdapter
+                                                                  -> OpenAIChatGenerator
+```
+
+The generator is built for DeepSeek's OpenAI-compatible endpoint from
+``DEEPSEEK_API_KEY``/``DEEPSEEK_MODEL`` when configured, or omitted entirely
+(prompt-only pipeline). Refinement is post-retrieval only: it never changes
+retrieval, indexing, or generation semantics.
+"""
+
+import os
+from collections.abc import Mapping
+from typing import Any, cast
+
+from haystack import Document, Pipeline, component
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
+from haystack.core.component.component import Component
+from haystack.dataclasses import ChatMessage
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils import Secret
+
+from ragrefine.integrations.haystack.component import (
+    RagRefineComponent,
+    refiner_for_profile,
+)
+from ragrefine.integrations.haystack.fixture import FROZEN_CORPUS
+from ragrefine.integrations.haystack.prompt import prompt_builder
+
+DEEPSEEK_BASE_URL = "https://api.deepseek.com/v1"
+
+
+@component
+class ChatPromptAdapter:
+    """Turn a PromptBuilder ``prompt`` string into a chat ``messages`` list."""
+
+    @component.output_types(messages=list[ChatMessage])
+    def run(self, prompt: str) -> dict[str, Any]:
+        """Wrap one prompt string as a user chat message."""
+        return {"messages": [ChatMessage.from_user(prompt)]}
+
+
+def demo_documents() -> list[Document]:
+    """Return the frozen retrieval corpus as documents for an example store."""
+    return [
+        Document(
+            id=str(row["id"]),
+            content=str(row["content"]),
+            meta=dict(cast(Mapping[str, Any], row["meta"])),
+        )
+        for row in FROZEN_CORPUS
+    ]
+
+
+def build_rag_pipeline(
+    document_store: InMemoryDocumentStore,
+    *,
+    top_k: int = 3,
+    profile: str = "b2-l-lexical",
+    api_key: str | None = None,
+    model: str | None = None,
+    generator: Component | None = None,
+) -> Pipeline:
+    """Return a retriever -> refine -> prompt -> chat-generator Haystack pipeline.
+
+    ``profile`` is any registered refiner profile (built-in ``b0`` and
+    ``b2-l-lexical``). When ``generator`` is supplied it is used as the final
+    chat generator; otherwise one is built for DeepSeek's OpenAI-compatible
+    endpoint when ``api_key`` is available, and the pipeline stops after the
+    prompt builder when it is not.
+    """
+    pipeline = Pipeline()
+    pipeline.add_component(
+        "retriever", InMemoryBM25Retriever(document_store=document_store)
+    )
+    pipeline.add_component(
+        "refine",
+        RagRefineComponent(top_k=top_k, refiner=refiner_for_profile(profile)),
+    )
+    pipeline.add_component("prompt", prompt_builder())
+    pipeline.connect("retriever.documents", "refine.documents")
+    pipeline.connect("refine.documents", "prompt.documents")
+    if generator is None:
+        if not api_key:
+            return pipeline
+        generator = OpenAIChatGenerator(
+            api_key=Secret.from_token(api_key),
+            model=model or os.environ.get("DEEPSEEK_MODEL", "deepseek-chat"),
+            api_base_url=DEEPSEEK_BASE_URL,
+            generation_kwargs={"temperature": 0},
+            max_retries=0,
+        )
+    pipeline.add_component("chat", ChatPromptAdapter())
+    pipeline.add_component("generator", generator)
+    pipeline.connect("prompt.prompt", "chat.prompt")
+    pipeline.connect("chat.messages", "generator.messages")
+    return pipeline

ragrefine/integrations/haystack/prompt.py

@@ -0,0 +1,17 @@
+"""Prompt construction helpers for the optional Haystack integration."""
+
+from haystack.components.builders import PromptBuilder
+
+PROMPT_TEMPLATE = """Question: {{ query }}
+Context:
+{% for document in documents %}
+[{{ document.id }}] {{ document.content }} | {{ document.meta }}
+{% endfor %}"""
+
+
+def prompt_builder() -> PromptBuilder:
+    """Create the fixed prompt builder shared by bypass and refinement paths."""
+    return PromptBuilder(
+        template=PROMPT_TEMPLATE,
+        required_variables=["query", "documents"],
+    )

ragrefine/models.py

@@ -0,0 +1,52 @@
+"""Core retriever-agnostic candidate domain models."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class Candidate:
+    """One immutable evidence unit supplied by an external retriever.
+
+    The model preserves the retriever's identifier, text, metadata, and optional
+    retrieval annotations without depending on any retrieval framework.
+    """
+
+    id: str
+    text: str
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+    retrieval_score: float | None = None
+    retrieval_rank: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CandidateSet:
+    """An immutable, ordered candidate collection from one named retriever."""
+
+    name: str
+    candidates: tuple[Candidate, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class RankingSignal:
+    """One optional, traceable ranking observation for a candidate."""
+
+    rank: int | None = None
+    score: float | None = None
+    details: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class RefinedCandidate:
+    """A candidate with its final ranking position and named evidence.
+
+    The wrapped candidate is always supplied by the caller; refinement never
+    creates or rewrites evidence. Retrieval provenance remains on ``candidate``.
+    ``rank`` is the position after ranking; a later context-selection position
+    is recorded separately in :class:`ragrefine.selection.SelectionRecord`.
+    """
+
+    candidate: Candidate
+    rank: int
+    signals: Mapping[str, RankingSignal] = field(default_factory=dict)

ragrefine/query/__init__.py

@@ -0,0 +1,18 @@
+"""Query-analysis helpers."""
+
+from ragrefine.query.normalize import normalize_text, normalized_terms
+from ragrefine.query.patterns import (
+    PatternMatch,
+    PatternRegistry,
+    PatternRule,
+    example_pattern_rules,
+)
+
+__all__ = [
+    "PatternMatch",
+    "PatternRegistry",
+    "PatternRule",
+    "example_pattern_rules",
+    "normalize_text",
+    "normalized_terms",
+]

ragrefine/query/normalize.py

@@ -0,0 +1,34 @@
+"""Deterministic, dependency-free text normalization for query analysis."""
+
+import re
+import unicodedata
+
+_WHITESPACE = re.compile(r"\s+")
+_TOKEN = re.compile(r"[^\W_]+(?:[-_][^\W_]+)*", flags=re.UNICODE)
+
+
+def normalize_text(text: str) -> str:
+    """Return a NFKC-normalized, case-folded, whitespace-normalized string.
+
+    The returned value is suitable for deterministic comparison.  It does not
+    stem, lemmatize, remove stop words, rewrite text, or mutate the caller's
+    original query/candidate text.
+    """
+    return _WHITESPACE.sub(" ", unicodedata.normalize("NFKC", text).casefold()).strip()
+
+
+def normalized_terms(text: str) -> tuple[str, ...]:
+    """Extract unique normalized lexical terms in their first-seen order.
+
+    Hyphenated and underscored technical identifiers remain one term.  The
+    uniqueness matches lexical coverage's set-based definition while ordering
+    makes evidence deterministic and human-readable.
+    """
+    terms: list[str] = []
+    seen: set[str] = set()
+    for match in _TOKEN.finditer(normalize_text(text)):
+        term = match.group().replace("_", "-")
+        if term not in seen:
+            terms.append(term)
+            seen.add(term)
+    return tuple(terms)