patternmem-rag 0.1.1__py3-none-any.whl

patternmem/eval_router.py

@@ -0,0 +1,316 @@
+"""
+patternmem.eval_router
+~~~~~~~~~~~~~~~~~~~~~~~
+EvalRouter — the *only* permitted translation boundary between external eval
+libraries (RAGAS, DeepEval) and the rest of the PatternMem system.
+
+Invariant 7: ``EvalRouter`` output is **always** a ``list[FailureSignal]``.
+No code outside this module ever touches a raw RAGAS or DeepEval object.
+
+Behaviour summary
+-----------------
+1. **Short-circuit**: if the pipeline output already contains a ``"reflect"``
+   key or a ``FaithfulnessEvaluator`` result dict, that score is used directly
+   and neither RAGAS nor DeepEval is invoked.
+2. **``eval="auto"``**: tries RAGAS, falls back to DeepEval, falls back to
+   ``"none"`` mode if neither is installed.
+3. **``eval="none"``**: returns a single ``FailureSignal`` with
+   ``FailureType.UNKNOWN`` and ``score=0.0`` — no external calls.
+4. **Compound failure**: when both ``faithfulness < LOW_THRESHOLD`` and
+   ``context_precision < LOW_THRESHOLD``, returns **two** ``FailureSignal``s
+   (one ``HALLUCINATION``, one ``RETRIEVAL_MISS``).
+5. Return type is always ``list[FailureSignal]`` — length 1 for single
+   failures, 2 for compound.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Literal
+
+from patternmem.types import FailureSignal, FailureType
+
+# ---------------------------------------------------------------------------
+# Numeric thresholds (module-level constants, not magic numbers)
+# ---------------------------------------------------------------------------
+
+LOW_THRESHOLD: float = 0.4   # below this → metric is "low"
+HIGH_THRESHOLD: float = 0.7  # above this → metric is "high"
+
+# ---------------------------------------------------------------------------
+# Internal helpers — RAGAS adapter
+# ---------------------------------------------------------------------------
+
+
+def _try_ragas(
+    query: str,
+    chunks: list[str],
+    answer: str,
+) -> tuple[float, float] | None:
+    """Return ``(faithfulness, context_precision)`` via RAGAS, or ``None`` if unavailable."""
+    try:
+        from ragas import evaluate as ragas_evaluate
+        from ragas.metrics import faithfulness, context_precision
+        from datasets import Dataset
+
+        ds = Dataset.from_dict(
+            {
+                "question": [query],
+                "answer": [answer],
+                "contexts": [chunks],
+            }
+        )
+        result = ragas_evaluate(ds, metrics=[faithfulness, context_precision])
+        faith = float(result["faithfulness"])
+        cp = float(result["context_precision"])
+        return faith, cp
+    except Exception:  # noqa: BLE001
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers — DeepEval adapter
+# ---------------------------------------------------------------------------
+
+
+def _try_deepeval(
+    query: str,
+    chunks: list[str],
+    answer: str,
+) -> tuple[float, float] | None:
+    """Return ``(faithfulness, context_precision)`` via DeepEval, or ``None``."""
+    try:
+        from deepeval.metrics import (
+            FaithfulnessMetric,
+            ContextualPrecisionMetric,
+        )
+        from deepeval.test_case import LLMTestCase
+
+        test_case = LLMTestCase(
+            input=query,
+            actual_output=answer,
+            retrieval_context=chunks,
+        )
+        fm = FaithfulnessMetric(threshold=0.5)
+        cm = ContextualPrecisionMetric(threshold=0.5)
+        fm.measure(test_case)
+        cm.measure(test_case)
+        return float(fm.score), float(cm.score)
+    except Exception:  # noqa: BLE001
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Short-circuit helper
+# ---------------------------------------------------------------------------
+
+
+def _extract_existing_reflection(
+    pipeline_output: Any,
+) -> tuple[float, float] | None:
+    """Return ``(faithfulness, context_precision)`` from an existing reflection key.
+
+    Checks for:
+    - ``pipeline_output["reflect"]`` — a float score assumed to be faithfulness
+    - ``pipeline_output["faithfulness_eval"]`` — LlamaIndex FaithfulnessEvaluator result
+
+    If found, sets ``context_precision = 1.0`` (no retrieval signal available from
+    reflection node) so classification routes to a generation-type failure only.
+    """
+    if not isinstance(pipeline_output, dict):
+        return None
+
+    # Key "reflect" — a raw float (0–1) indicating faithfulness
+    if "reflect" in pipeline_output:
+        score = pipeline_output["reflect"]
+        try:
+            return float(score), 1.0
+        except (TypeError, ValueError):
+            pass
+
+    # LlamaIndex FaithfulnessEvaluator result — dict with "score" key
+    if "faithfulness_eval" in pipeline_output:
+        fe = pipeline_output["faithfulness_eval"]
+        if isinstance(fe, dict) and "score" in fe:
+            try:
+                return float(fe["score"]), 1.0
+            except (TypeError, ValueError):
+                pass
+
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Classification matrix
+# ---------------------------------------------------------------------------
+
+
+def _classify(
+    faithfulness: float,
+    context_precision: float,
+    query: str,
+    hint_override: str = "",
+) -> list[FailureSignal]:
+    """Map (faithfulness, context_precision) → list[FailureSignal] per the spec matrix.
+
+    Compound case: both low → two independent signals.
+    """
+    faith_low = faithfulness < LOW_THRESHOLD
+    cp_low = context_precision < LOW_THRESHOLD
+    faith_high = faithfulness >= HIGH_THRESHOLD
+    cp_high = context_precision >= HIGH_THRESHOLD
+
+    signals: list[FailureSignal] = []
+    trace_id = str(uuid.uuid4())
+
+    if faith_low and cp_low:
+        # Compound case — two independent signals
+        signals.append(
+            FailureSignal(
+                score=faithfulness,
+                failure_type=FailureType.HALLUCINATION,
+                root_cause="Both faithfulness and context precision are low; "
+                           "generation is not grounded in retrieved context.",
+                hint="Add explicit grounding instructions to the generator; "
+                     "check that context is attached to the prompt.",
+                trace_id=trace_id,
+            )
+        )
+        signals.append(
+            FailureSignal(
+                score=context_precision,
+                failure_type=FailureType.RETRIEVAL_MISS,
+                root_cause="Context precision is critically low; retrieved chunks "
+                           "are not relevant to the query.",
+                hint="Broaden retrieval query; increase top-k; check index freshness.",
+                trace_id=str(uuid.uuid4()),
+            )
+        )
+    elif faith_low and not cp_low:
+        # Retrieval found relevant docs but generator drifted
+        ft = (
+            FailureType.FAITHFULNESS_DRIFT
+            if faithfulness >= 0.2  # partially faithful
+            else FailureType.HALLUCINATION
+        )
+        signals.append(
+            FailureSignal(
+                score=faithfulness,
+                failure_type=ft,
+                root_cause="Generator answer diverges from retrieved context despite "
+                           "relevant retrieval.",
+                hint="Add citation constraints; reduce temperature; force CoT.",
+                trace_id=trace_id,
+            )
+        )
+    elif not faith_low and cp_low:
+        # Generator is faithful to what it has, but retrieval quality is poor
+        ft = FailureType.RETRIEVAL_MISS if context_precision < 0.2 else FailureType.RETRIEVAL_RANK
+        signals.append(
+            FailureSignal(
+                score=context_precision,
+                failure_type=ft,
+                root_cause="Retrieved context has low precision relative to the query.",
+                hint="Refine retrieval query; add MMR re-ranking; expand synonyms.",
+                trace_id=trace_id,
+            )
+        )
+    else:
+        # Both high → no actionable failure
+        signals.append(
+            FailureSignal(
+                score=min(faithfulness, context_precision),
+                failure_type=FailureType.UNKNOWN,
+                root_cause="No clear failure detected.",
+                hint="",
+                trace_id=trace_id,
+            )
+        )
+
+    return signals
+
+
+# ---------------------------------------------------------------------------
+# Public class
+# ---------------------------------------------------------------------------
+
+
+class EvalRouter:
+    """Normalises RAGAS and DeepEval outputs into ``FailureSignal`` lists.
+
+    Parameters
+    ----------
+    mode:
+        One of ``"ragas"``, ``"deepeval"``, ``"auto"``, ``"none"``.
+        ``"auto"`` tries RAGAS → DeepEval → none in order.
+    """
+
+    def __init__(self, mode: Literal["ragas", "deepeval", "auto", "none"]) -> None:
+        self._mode = mode
+
+    def evaluate(
+        self,
+        query: str,
+        chunks: list[str],
+        answer: str,
+        pipeline_output: Any = None,
+    ) -> list[FailureSignal]:
+        """Evaluate a single query/answer/context triple.
+
+        Parameters
+        ----------
+        query:
+            The original user query.
+        chunks:
+            Retrieved context chunks passed to the generator.
+        answer:
+            The generated answer.
+        pipeline_output:
+            Raw return value from the pipeline call.  Inspected for existing
+            reflection keys before invoking any eval library.
+
+        Returns
+        -------
+        list[FailureSignal]
+            Length 1 for single failures; length 2 for compound (both faithfulness
+            and context_precision low).  Always returns at least one signal.
+        """
+        # --- Short-circuit: existing reflection in pipeline output ---
+        existing = _extract_existing_reflection(pipeline_output)
+        if existing is not None:
+            faith, cp = existing
+            return _classify(faith, cp, query)
+
+        # --- eval="none" ---
+        if self._mode == "none":
+            return [
+                FailureSignal(
+                    score=0.0,
+                    failure_type=FailureType.UNKNOWN,
+                    root_cause="Evaluation disabled (eval='none').",
+                    hint="",
+                )
+            ]
+
+        # --- Try the configured adapter(s) ---
+        scores: tuple[float, float] | None = None
+
+        if self._mode in ("ragas", "auto"):
+            scores = _try_ragas(query, chunks, answer)
+
+        if scores is None and self._mode in ("deepeval", "auto"):
+            scores = _try_deepeval(query, chunks, answer)
+
+        if scores is None:
+            # Neither library available — fall back to none behaviour
+            return [
+                FailureSignal(
+                    score=0.0,
+                    failure_type=FailureType.UNKNOWN,
+                    root_cause="No evaluation library available (RAGAS and DeepEval not installed).",
+                    hint="",
+                )
+            ]
+
+        faith, cp = scores
+        return _classify(faith, cp, query)

patternmem/middleware.py

@@ -0,0 +1,336 @@
+"""
+patternmem.middleware
+~~~~~~~~~~~~~~~~~~~~~~
+PatternMemMiddleware — the public one-line integration point.
+
+Usage::
+
+    from patternmem import PatternMemMiddleware
+
+    middleware = PatternMemMiddleware(pipeline=my_rag_pipeline)
+    answer = await middleware.ainvoke("What is the capital of France?")
+
+    # Or as an async context manager (recommended for lifecycle management):
+    async with PatternMemMiddleware(pipeline=my_rag_pipeline) as m:
+        answer = await m.ainvoke("What is the capital of France?")
+
+The 3-phase loop
+----------------
+Phase 1 (sync, ~50 ms target):
+    Embed query (MiniLM) → lookup patterns → augment → call pipeline → return.
+    The caller receives the answer *immediately* upon pipeline return.
+
+Phase 2 (async background, non-blocking):
+    EvalRouter evaluates (query, chunks, answer).
+    FailureSignal(s) are enqueued into the BackgroundReflector queue.
+
+Phase 3 (async background, queue consumer):
+    BackgroundReflector dequeues → LLM extracts pattern → backend write/decay.
+
+Invariants enforced here
+------------------------
+- Invariant 1: LLMResolverError is raised at __init__, not at ainvoke time.
+- Invariant 3: ainvoke() returns as soon as the pipeline returns.
+- Invariant 5: backend="json", eval="none", observability=None works with zero deps.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+from typing import Any, Callable, Literal, Optional
+
+from sentence_transformers import SentenceTransformer
+
+from patternmem.augmenter import Augmenter
+from patternmem.backend import MemoryBackend
+from patternmem.eval_router import EvalRouter
+from patternmem.observability import ObservabilityEmitter
+from patternmem.reflector import BackgroundReflector
+from patternmem.resolver import LLMResolver
+from patternmem.types import FailurePattern, FailureSignal
+
+logger = logging.getLogger(__name__)
+
+# MiniLM model name — fully local, no API key
+_EMBED_MODEL = "all-MiniLM-L6-v2"
+
+
+def _build_backend(
+    kind: str,
+    similarity_threshold: float,
+) -> MemoryBackend:
+    """Instantiate the requested backend with sensible defaults."""
+    if kind == "json":
+        from patternmem.backends.json_backend import JSONBackend
+        return JSONBackend(similarity_threshold=similarity_threshold)
+    if kind == "sqlite":
+        from patternmem.backends.sqlite_backend import SQLiteBackend
+        return SQLiteBackend(similarity_threshold=similarity_threshold)
+    if kind == "networkx":
+        from patternmem.backends.networkx_backend import NetworkXBackend
+        return NetworkXBackend(similarity_threshold=similarity_threshold)
+    if kind == "chroma":
+        from patternmem.backends.chroma_backend import ChromaBackend
+        return ChromaBackend(similarity_threshold=similarity_threshold)
+    if kind == "faiss":
+        from patternmem.backends.faiss_backend import FAISSBackend
+        return FAISSBackend(similarity_threshold=similarity_threshold)
+    if kind == "neo4j":
+        raise ValueError(
+            "Neo4jBackend requires additional configuration (uri, auth). "
+            "Instantiate it directly and pass a MemoryBackend instance instead."
+        )
+    raise ValueError(
+        f"Unknown backend: {kind!r}. "
+        "Choose from: json, sqlite, networkx, chroma, faiss, neo4j."
+    )
+
+
+def _extract_chunks(pipeline_output: Any) -> list[str]:
+    """Best-effort extraction of retrieved context chunks from pipeline output."""
+    if isinstance(pipeline_output, dict):
+        for key in ("chunks", "context", "source_documents", "contexts"):
+            val = pipeline_output.get(key)
+            if isinstance(val, list):
+                return [str(v) for v in val]
+    return []
+
+
+def _extract_answer(pipeline_output: Any) -> str:
+    """Best-effort extraction of the generated answer from pipeline output."""
+    if isinstance(pipeline_output, str):
+        return pipeline_output
+    if isinstance(pipeline_output, dict):
+        for key in ("answer", "result", "output", "response", "text"):
+            val = pipeline_output.get(key)
+            if isinstance(val, str):
+                return val
+    return str(pipeline_output)
+
+
+class PatternMemMiddleware:
+    """Framework-agnostic RAG middleware with persistent failure-pattern memory.
+
+    Parameters
+    ----------
+    pipeline:
+        Any callable that accepts a query string (and optional kwargs) and
+        returns an answer.  May be synchronous or asynchronous.
+    llm:
+        Optional explicit LLM.  If ``None``, ``LLMResolver`` probes the
+        pipeline for a detectable LLM.  Raises ``LLMResolverError`` at init
+        if neither succeeds.
+    backend:
+        Storage backend identifier, or an existing ``MemoryBackend`` instance.
+        Accepts ``"json"`` (default), ``"sqlite"``, ``"networkx"``.
+        For Neo4j, pass an instantiated ``Neo4jBackend`` object directly.
+    eval:
+        Evaluation mode.  ``"auto"`` (default) tries RAGAS then DeepEval then
+        falls back to ``"none"`` behaviour.
+    observability:
+        ``"langfuse"``, ``"otel"``, or ``None`` (default, no-op).
+    similarity_threshold:
+        Cosine similarity floor for Phase 1 pattern lookup.
+    allow_param_override:
+        If ``True``, Augmenter may set ``llm_call_kwargs`` (e.g. temperature).
+    rewrite_feedback:
+        If ``True``, Augmenter adds a ``rewrite_constraint`` key.
+
+    Raises
+    ------
+    LLMResolverError
+        At construction time if the LLM cannot be resolved.
+    """
+
+    def __init__(
+        self,
+        pipeline: Callable[..., Any],
+        llm: Optional[Any] = None,
+        backend: "Literal['neo4j', 'sqlite', 'json', 'networkx', 'chroma', 'faiss'] | MemoryBackend" = "json",
+        eval: Literal["ragas", "deepeval", "auto", "none"] = "auto",
+        observability: Optional[Literal["langfuse", "otel"]] = None,
+        similarity_threshold: float = 0.82,
+        allow_param_override: bool = False,
+        rewrite_feedback: bool = False,
+    ) -> None:
+        self._pipeline = pipeline
+        self._similarity_threshold = similarity_threshold
+
+        # --- Backend ---
+        if isinstance(backend, MemoryBackend):
+            self._backend: MemoryBackend = backend
+        else:
+            self._backend = _build_backend(str(backend), similarity_threshold)
+
+        # --- LLM Resolution (Invariant 1: fail here, not at runtime) ---
+        # When eval="none", we still attempt resolution so the user gets an
+        # early error if they forget llm=.  They can pass llm=None explicitly
+        # only when building a test double.
+        self._resolver = LLMResolver(pipeline=pipeline, llm=llm)
+
+        # --- EvalRouter ---
+        self._eval_router = EvalRouter(mode=eval)
+
+        # --- Augmenter ---
+        self._augmenter = Augmenter(
+            pipeline=pipeline,
+            rewrite_feedback=rewrite_feedback,
+            allow_param_override=allow_param_override,
+        )
+
+        # --- Observability ---
+        self._emitter = ObservabilityEmitter(mode=observability)
+
+        # --- BackgroundReflector (started lazily on first ainvoke) ---
+        self._reflector = BackgroundReflector(
+            backend=self._backend,
+            llm=self._resolver.llm,
+            emitter=self._emitter,
+        )
+
+        # --- Embedding model (MiniLM, fully local) ---
+        self._encoder: SentenceTransformer | None = None  # lazy init
+
+    def _get_encoder(self) -> SentenceTransformer:
+        if self._encoder is None:
+            # token=False forces anonymous download, bypassing any cached/expired
+            # HuggingFace tokens.  The model is freely public.
+            self._encoder = SentenceTransformer(_EMBED_MODEL, token=False)
+        return self._encoder
+
+    def _embed(self, text: str) -> list[float]:
+        enc = self._get_encoder()
+        vec = enc.encode(text, normalize_embeddings=True)
+        result: list[float] = vec.tolist()
+        return result
+
+    # ------------------------------------------------------------------
+    # Async context manager
+    # ------------------------------------------------------------------
+
+    async def __aenter__(self) -> "PatternMemMiddleware":
+        self._reflector.start()
+        return self
+
+    async def __aexit__(self, *_: Any) -> None:
+        await self._reflector.stop()
+
+    # ------------------------------------------------------------------
+    # Core public method
+    # ------------------------------------------------------------------
+
+    async def ainvoke(self, query: str, **kwargs: Any) -> Any:
+        """Invoke the wrapped pipeline with optional pattern-based augmentation.
+
+        Phase 1 runs synchronously in this coroutine; Phases 2 and 3 are fired
+        as background tasks.  The caller receives the pipeline's answer as soon
+        as it is available (Invariant 3).
+
+        Parameters
+        ----------
+        query:
+            The user's natural-language query.
+        **kwargs:
+            Additional keyword arguments forwarded to the pipeline.
+
+        Returns
+        -------
+        Any
+            Whatever the wrapped pipeline returns.
+        """
+        # Ensure the reflector is running
+        self._reflector.start()
+
+        # ------------------------------------------------------------------
+        # Phase 1 — Embed → Lookup → Augment → Call pipeline
+        # ------------------------------------------------------------------
+        with self._emitter.span("patternmem.phase1", {"query": query[:200]}):
+            query_embedding = await asyncio.to_thread(self._embed, query)
+
+            matched_patterns: list[FailurePattern] = await self._backend.lookup_patterns(
+                query_embedding, top_k=3
+            )
+
+            if matched_patterns:
+                augmented = self._augmenter.build(matched_patterns)
+                logger.debug(
+                    "Phase 1 HIT: %d patterns matched, augmented keys: %s",
+                    len(matched_patterns),
+                    list(augmented.keys()),
+                )
+                pipeline_kwargs = {**kwargs, **augmented}
+            else:
+                logger.debug("Phase 1 MISS: no patterns above threshold")
+                pipeline_kwargs = kwargs
+
+            # Call the pipeline (sync, async function, or async callable instance)
+            if inspect.iscoroutinefunction(self._pipeline) or inspect.iscoroutinefunction(
+                getattr(self._pipeline, "__call__", None)
+            ):
+                pipeline_output = await self._pipeline(query, **pipeline_kwargs)
+            else:
+                pipeline_output = await asyncio.to_thread(
+                    self._pipeline, query, **pipeline_kwargs
+                )
+
+        # Return to caller immediately (Invariant 3)
+        # ------------------------------------------------------------------
+        # Phase 2 — Background eval → enqueue to reflector
+        # ------------------------------------------------------------------
+        asyncio.create_task(
+            self._phase2(
+                query=query,
+                query_embedding=query_embedding,
+                pipeline_output=pipeline_output,
+                matched_patterns=matched_patterns,
+            ),
+            name="patternmem.phase2",
+        )
+
+        return pipeline_output
+
+    # ------------------------------------------------------------------
+    # Phase 2 (background)
+    # ------------------------------------------------------------------
+
+    async def _phase2(
+        self,
+        query: str,
+        query_embedding: list[float],
+        pipeline_output: Any,
+        matched_patterns: list[FailurePattern],
+    ) -> None:
+        """Run EvalRouter in background and enqueue signals for Phase 3."""
+        try:
+            with self._emitter.span("patternmem.phase2", {"query": query[:200]}):
+                chunks = _extract_chunks(pipeline_output)
+                answer = _extract_answer(pipeline_output)
+
+                signals: list[FailureSignal] = await asyncio.to_thread(
+                    self._eval_router.evaluate,
+                    query,
+                    chunks,
+                    answer,
+                    pipeline_output,
+                )
+
+                for signal in signals:
+                    # Match signal to an existing pattern by failure_type
+                    matched = next(
+                        (
+                            p
+                            for p in matched_patterns
+                            if p.failure_type == signal.failure_type
+                        ),
+                        None,
+                    )
+                    # matched=None → new pattern will be written in Phase 3
+                    self._reflector.enqueue(
+                        signal=signal,
+                        query_embedding=query_embedding,
+                        matched_pattern=matched,
+                    )
+        except Exception:  # noqa: BLE001
+            logger.exception("Phase 2 background task failed")