patternmem-rag 0.1.1__py3-none-any.whl

patternmem/observability.py

@@ -0,0 +1,101 @@
+"""
+patternmem.observability
+~~~~~~~~~~~~~~~~~~~~~~~~~
+Thin, no-op-safe wrappers around Langfuse and OpenTelemetry.
+
+Design rules
+------------
+- If the requested observability library is not installed, every function
+  silently becomes a no-op.  This keeps zero-credential mode working with
+  ``observability=None`` (Invariant 5).
+- No observability call may block ``ainvoke()`` (Invariant 3).  All emits
+  happen inside the already-async background Phase 3 task.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Any, Generator, Literal, Optional
+
+
+ObservabilityMode = Optional[Literal["langfuse", "otel"]]
+
+
+class ObservabilityEmitter:
+    """Emits trace spans and scores to the configured observability backend.
+
+    Parameters
+    ----------
+    mode:
+        ``"langfuse"``, ``"otel"``, or ``None`` (no-op).
+    """
+
+    def __init__(self, mode: ObservabilityMode = None) -> None:
+        self._mode = mode
+        self._langfuse: Any = None
+        self._tracer: Any = None
+
+        if mode == "langfuse":
+            self._langfuse = self._init_langfuse()
+        elif mode == "otel":
+            self._tracer = self._init_otel()
+
+    # ------------------------------------------------------------------
+    # Init helpers (no-op on ImportError)
+    # ------------------------------------------------------------------
+
+    def _init_langfuse(self) -> Any:
+        try:
+            from langfuse import Langfuse
+            return Langfuse()
+        except ImportError:
+            return None
+
+    def _init_otel(self) -> Any:
+        try:
+            from opentelemetry import trace
+            return trace.get_tracer("patternmem")
+        except ImportError:
+            return None
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def emit_score(
+        self,
+        trace_id: str,
+        score: float,
+        failure_type: str,
+        name: str = "patternmem.failure_score",
+    ) -> None:
+        """Emit a score observation (non-blocking; called from Phase 3 background)."""
+        if self._mode == "langfuse" and self._langfuse is not None:
+            try:
+                self._langfuse.score(
+                    trace_id=trace_id,
+                    name=name,
+                    value=score,
+                    comment=f"failure_type={failure_type}",
+                )
+            except Exception:  # noqa: BLE001
+                pass
+
+    @contextmanager
+    def span(
+        self,
+        name: str,
+        attributes: dict[str, Any] | None = None,
+    ) -> Generator[Any, None, None]:
+        """Context manager that wraps a code block in a trace span.
+
+        No-op if observability is disabled or the library is not installed.
+        """
+        if self._mode == "otel" and self._tracer is not None:
+            with self._tracer.start_as_current_span(name) as span:
+                if attributes:
+                    for k, v in attributes.items():
+                        span.set_attribute(k, str(v))
+                yield span
+        else:
+            yield None

patternmem/reflector.py

@@ -0,0 +1,211 @@
+"""
+patternmem.reflector
+~~~~~~~~~~~~~~~~~~~~~
+BackgroundReflector — asyncio queue consumer for Phase 3 of the 3-phase loop.
+
+Responsibilities
+----------------
+1. Dequeue ``(FailureSignal, query, chunks, answer, matched_pattern_id_or_None)``
+   tuples from the internal asyncio queue.
+2. Use the resolved LLM to extract a ``FailurePattern`` from the signal.
+3. Write the pattern to the backend (or update+decay an existing one).
+4. Emit the score to the configured observability backend.
+
+Lifecycle
+---------
+- Started as an ``asyncio.Task`` on the first ``ainvoke()`` call.
+- Cancelled and drained when ``PatternMemMiddleware.__aexit__`` is called.
+- Never blocks ``ainvoke()`` (Invariant 3).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from patternmem.backend import MemoryBackend
+from patternmem.decay import compute_new_decay_weight, should_evict
+from patternmem.observability import ObservabilityEmitter
+from patternmem.types import FailurePattern, FailureSignal
+
+logger = logging.getLogger(__name__)
+
+_SENTINEL = object()  # Signals queue shutdown
+
+
+@dataclass
+class _ReflectorJob:
+    signal: FailureSignal
+    query_embedding: list[float]
+    matched_pattern: Optional[FailurePattern]  # None if Phase 1 was a miss
+
+
+class BackgroundReflector:
+    """Async queue consumer that persists failure patterns to the backend.
+
+    Parameters
+    ----------
+    backend:
+        The storage backend to write/update/delete patterns in.
+    llm:
+        The resolved LLM.  Used to extract ``root_cause`` and ``hint_text``
+        from the raw ``FailureSignal``.
+    emitter:
+        Observability emitter.  No-op if observability is disabled.
+    """
+
+    def __init__(
+        self,
+        backend: MemoryBackend,
+        llm: Any,
+        emitter: ObservabilityEmitter,
+    ) -> None:
+        self._backend = backend
+        self._llm = llm
+        self._emitter = emitter
+        self._queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=1000)
+        self._task: asyncio.Task[None] | None = None
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def start(self) -> None:
+        """Start the background consumer task."""
+        if self._task is None or self._task.done():
+            self._task = asyncio.create_task(self._consume(), name="patternmem.reflector")
+
+    async def stop(self) -> None:
+        """Signal shutdown and wait for the queue to drain."""
+        await self._queue.put(_SENTINEL)
+        if self._task is not None:
+            try:
+                await asyncio.wait_for(self._task, timeout=5.0)
+            except asyncio.TimeoutError:
+                self._task.cancel()
+
+    # ------------------------------------------------------------------
+    # Enqueue (called from Phase 2 background task in middleware)
+    # ------------------------------------------------------------------
+
+    def enqueue(
+        self,
+        signal: FailureSignal,
+        query_embedding: list[float],
+        matched_pattern: FailurePattern | None = None,
+    ) -> None:
+        """Non-blocking enqueue — never raises even if the queue is full."""
+        try:
+            self._queue.put_nowait(
+                _ReflectorJob(
+                    signal=signal,
+                    query_embedding=query_embedding,
+                    matched_pattern=matched_pattern,
+                )
+            )
+        except asyncio.QueueFull:
+            logger.warning("PatternMem reflector queue is full; dropping signal %s", signal.trace_id)
+
+    # ------------------------------------------------------------------
+    # Consumer loop
+    # ------------------------------------------------------------------
+
+    async def _consume(self) -> None:
+        while True:
+            item = await self._queue.get()
+            if item is _SENTINEL:
+                self._queue.task_done()
+                break
+            try:
+                await self._process(item)
+            except Exception:  # noqa: BLE001
+                logger.exception("BackgroundReflector error processing job")
+            finally:
+                self._queue.task_done()
+
+    async def _process(self, job: _ReflectorJob) -> None:
+        signal = job.signal
+        matched = job.matched_pattern
+
+        if matched is not None:
+            # --- Pattern exists: apply decay or reinforce ---
+            new_weight = compute_new_decay_weight(
+                stored=matched.decay_weight,
+                old_score=matched.score,
+                new_score=signal.score,
+            )
+            if should_evict(new_weight):
+                logger.debug("Evicting pattern %s (weight %.3f)", matched.id, new_weight)
+                try:
+                    await self._backend.delete_pattern(matched.id)
+                except KeyError:
+                    pass  # Already deleted — idempotent
+            else:
+                logger.debug(
+                    "Updating pattern %s decay_weight %.3f → %.3f",
+                    matched.id,
+                    matched.decay_weight,
+                    new_weight,
+                )
+                await self._backend.update_pattern(matched.id, new_weight)
+        else:
+            # --- New pattern: extract hint via LLM and write ---
+            root_cause, hint_text = await self._extract_from_llm(signal)
+
+            pattern = FailurePattern(
+                query_embedding=job.query_embedding,
+                failure_type=signal.failure_type,
+                root_cause=root_cause,
+                hint_text=hint_text,
+                score=signal.score,
+                decay_weight=1.0,
+            )
+            await self._backend.write_pattern(pattern)
+            logger.debug("Wrote new pattern %s (%s)", pattern.id, signal.failure_type.name)
+
+        # Emit observability score
+        self._emitter.emit_score(
+            trace_id=signal.trace_id,
+            score=signal.score,
+            failure_type=signal.failure_type.name,
+        )
+
+    async def _extract_from_llm(self, signal: FailureSignal) -> tuple[str, str]:
+        """Ask the LLM to refine root_cause and hint_text from the signal.
+
+        Falls back to the signal's own fields if LLM call fails.
+        """
+        if signal.hint:
+            # Signal already carries a hint (from the classification matrix)
+            return signal.root_cause, signal.hint
+
+        prompt = (
+            f"A RAG pipeline produced a failure of type '{signal.failure_type.name}'.\n"
+            f"Score: {signal.score:.3f}\n"
+            f"Root cause description: {signal.root_cause}\n\n"
+            "In one sentence each:\n"
+            "1. What is the root cause?\n"
+            "2. What retrieval or generation hint would prevent this failure next time?\n"
+            "Reply in JSON: {\"root_cause\": \"...\", \"hint\": \"...\"}"
+        )
+        try:
+            import json as _json
+
+            # Duck-typed LLM call — supports langchain .invoke() and raw callables
+            if hasattr(self._llm, "invoke"):
+                raw = self._llm.invoke(prompt)
+                if hasattr(raw, "content"):
+                    raw = raw.content
+            elif callable(self._llm):
+                raw = self._llm(prompt)
+            else:
+                raw = str(self._llm)
+
+            parsed = _json.loads(str(raw))
+            return str(parsed.get("root_cause", signal.root_cause)), str(
+                parsed.get("hint", signal.hint)
+            )
+        except Exception:  # noqa: BLE001
+            return signal.root_cause, signal.hint

patternmem/resolver.py

@@ -0,0 +1,138 @@
+"""
+patternmem.resolver
+~~~~~~~~~~~~~~~~~~~~
+LLMResolver — discovers a usable LLM from the wrapped pipeline at init time.
+
+Design rules
+------------
+- Duck-typing only.  No ``isinstance`` against any framework type.
+- No framework imports (langchain, llama_index, langgraph, etc.).
+- All attribute probes are wrapped in ``try/except AttributeError`` — probing
+  must never raise an unexpected exception.
+- If all probe paths fail and no ``llm=`` kwarg was supplied, raises
+  ``LLMResolverError`` immediately (Invariant 1: fail at init, not at runtime).
+
+Probe priority order (do not reorder)
+--------------------------------------
+1. ``pipeline.llm``
+2. ``pipeline.combine_docs_chain.llm_chain.llm``
+3. ``pipeline.nodes["generate"].llm``
+4. ``pipeline._llm``
+5. ``pipeline.as_query_engine()._llm``
+6. Explicit ``llm=`` parameter
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from patternmem.types import LLMResolverError
+
+# Human-readable descriptions of each probe path (used in error messages)
+_PROBE_DESCRIPTIONS = [
+    "pipeline.llm",
+    "pipeline.combine_docs_chain.llm_chain.llm",
+    'pipeline.nodes["generate"].llm',
+    "pipeline._llm",
+    "pipeline.as_query_engine()._llm",
+]
+
+
+def _probe(pipeline: Any) -> Any | None:
+    """Attempt all probe paths and return the first non-None result, or None."""
+
+    # Path 1: pipeline.llm (LangChain LLMChain, simple case)
+    try:
+        llm = pipeline.llm
+        if llm is not None:
+            return llm
+    except AttributeError:
+        pass
+
+    # Path 2: pipeline.combine_docs_chain.llm_chain.llm (LangChain RetrievalQA)
+    try:
+        llm = pipeline.combine_docs_chain.llm_chain.llm
+        if llm is not None:
+            return llm
+    except AttributeError:
+        pass
+
+    # Path 3: pipeline.nodes["generate"].llm (LangGraph compiled graph)
+    try:
+        nodes = pipeline.nodes
+        if isinstance(nodes, dict) and "generate" in nodes:
+            llm = nodes["generate"].llm
+            if llm is not None:
+                return llm
+    except AttributeError:
+        pass
+
+    # Path 4: pipeline._llm (LlamaIndex query engine)
+    try:
+        llm = pipeline._llm  # noqa: SLF001
+        if llm is not None:
+            return llm
+    except AttributeError:
+        pass
+
+    # Path 5: pipeline.as_query_engine()._llm (LlamaIndex index)
+    try:
+        engine = pipeline.as_query_engine()
+        llm = engine._llm  # noqa: SLF001
+        if llm is not None:
+            return llm
+    except (AttributeError, Exception):  # as_query_engine may raise
+        pass
+
+    return None
+
+
+class LLMResolver:
+    """Resolves and holds a reference to the LLM used by the wrapped pipeline.
+
+    Parameters
+    ----------
+    pipeline:
+        The user's pipeline object.  May be any callable or object — resolver
+        uses duck-typing only.
+    llm:
+        Explicit LLM override.  If provided, all probe paths are skipped.
+
+    Raises
+    ------
+    LLMResolverError
+        If no LLM can be resolved from the pipeline and ``llm`` is ``None``.
+        This is raised at construction time, never at call time.
+    """
+
+    def __init__(self, pipeline: Any, llm: Any | None = None) -> None:
+        if llm is not None:
+            self._llm = llm
+            self._source = "explicit llm= parameter"
+            return
+
+        resolved = _probe(pipeline)
+        if resolved is None:
+            raise LLMResolverError(
+                "PatternMemMiddleware could not resolve an LLM from the pipeline. "
+                "Tried probe paths:\n"
+                + "\n".join(f"  {i + 1}. {desc}" for i, desc in enumerate(_PROBE_DESCRIPTIONS))
+                + "\n\nFix: pass llm= explicitly:\n"
+                "  PatternMemMiddleware(pipeline=my_pipeline, llm=my_llm)"
+            )
+
+        self._llm = resolved
+        self._source = "auto-resolved from pipeline"
+
+    @property
+    def llm(self) -> Any:
+        """The resolved LLM object."""
+        return self._llm
+
+    @property
+    def source(self) -> str:
+        """Human-readable description of how the LLM was resolved."""
+        return self._source
+
+    def __repr__(self) -> str:
+        return f"LLMResolver(llm={self._llm!r}, source={self._source!r})"

patternmem/types.py

@@ -0,0 +1,212 @@
+"""
+patternmem.types
+~~~~~~~~~~~~~~~~
+Core data contracts for PatternMem RAG middleware.
+
+This module is the **only** place where ``FailureType``, ``FailureSignal``,
+and ``FailurePattern`` are defined.  Everything else in the codebase imports
+from here — never redefines these types locally.
+
+No logic lives here: only ``@dataclass`` / ``Enum`` definitions and docstrings.
+The module must remain importable with zero external dependencies.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum, auto
+
+
+
+# ---------------------------------------------------------------------------
+# FailureType
+# ---------------------------------------------------------------------------
+
+
+class FailureType(Enum):
+    """Taxonomy of RAG failure modes detected by ``EvalRouter``.
+
+    Members
+    -------
+    RETRIEVAL_MISS
+        The retriever returned no documents relevant to the query.
+    RETRIEVAL_RANK
+        Relevant documents were retrieved but ranked too low to influence the
+        generated answer.
+    RETRIEVAL_STALE
+        Retrieved documents are outdated relative to the query's time context.
+    RETRIEVAL_ENTITY_DROP
+        A key named entity in the query was dropped from the retrieved context.
+    HALLUCINATION
+        The generated answer contains facts not supported by retrieved context.
+    FAITHFULNESS_DRIFT
+        The generated answer is partially faithful but drifts from context in
+        meaningful ways (softer than outright hallucination).
+    REFUSAL
+        The generator refused to answer (e.g. safety filter, out-of-scope).
+    INCOMPLETE
+        The generator produced a truncated or partial answer.
+    UNKNOWN
+        Failure mode cannot be determined from available signals.  This is also
+        the sentinel value used by ``eval="none"`` mode.
+    """
+
+    RETRIEVAL_MISS = auto()
+    RETRIEVAL_RANK = auto()
+    RETRIEVAL_STALE = auto()
+    RETRIEVAL_ENTITY_DROP = auto()
+    HALLUCINATION = auto()
+    FAITHFULNESS_DRIFT = auto()
+    REFUSAL = auto()
+    INCOMPLETE = auto()
+    UNKNOWN = auto()
+
+
+# ---------------------------------------------------------------------------
+# Injection-lane classification helpers
+# ---------------------------------------------------------------------------
+
+#: Failure types that should route to the *retrieval* injection lane.
+RETRIEVAL_FAILURE_TYPES: frozenset[FailureType] = frozenset(
+    {
+        FailureType.RETRIEVAL_MISS,
+        FailureType.RETRIEVAL_RANK,
+        FailureType.RETRIEVAL_STALE,
+        FailureType.RETRIEVAL_ENTITY_DROP,
+    }
+)
+
+#: Failure types that should route to the *generation* injection lane.
+GENERATION_FAILURE_TYPES: frozenset[FailureType] = frozenset(
+    {
+        FailureType.HALLUCINATION,
+        FailureType.FAITHFULNESS_DRIFT,
+        FailureType.REFUSAL,
+        FailureType.INCOMPLETE,
+    }
+)
+
+
+def is_retrieval_failure(ft: FailureType) -> bool:
+    """Return ``True`` if *ft* should route to ``augmented_input["retrieval_hint"]``."""
+    return ft in RETRIEVAL_FAILURE_TYPES
+
+
+def is_generation_failure(ft: FailureType) -> bool:
+    """Return ``True`` if *ft* should route to ``augmented_input["generation_constraint"]``."""
+    return ft in GENERATION_FAILURE_TYPES
+
+
+# ---------------------------------------------------------------------------
+# LLMResolverError
+# ---------------------------------------------------------------------------
+
+
+class LLMResolverError(RuntimeError):
+    """Raised at ``PatternMemMiddleware.__init__`` time when no LLM can be resolved.
+
+    This is an *init-time* error, never a runtime error (see Invariant 1).
+    The message always includes which probe paths were attempted so the user
+    can quickly diagnose the issue.
+    """
+
+
+# ---------------------------------------------------------------------------
+# FailureSignal
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class FailureSignal:
+    """Normalised failure signal produced exclusively by ``EvalRouter``.
+
+    ``EvalRouter`` is the *only* permitted source of ``FailureSignal`` objects.
+    No other code in the codebase constructs them directly except in tests.
+
+    Attributes
+    ----------
+    score : float
+        Evaluation score in [0.0, 1.0].  Lower scores indicate worse quality.
+        Set to ``0.0`` when ``eval="none"`` mode is active.
+    failure_type : FailureType
+        Classified failure mode.  ``FailureType.UNKNOWN`` when the mode cannot
+        be determined.
+    root_cause : str
+        Human-readable description of *why* the failure occurred, extracted by
+        the ``BackgroundReflector`` using the resolved LLM.
+    hint : str
+        Actionable hint text to be injected in the next Phase 1 pass via the
+        appropriate augmentation lane.
+    trace_id : str
+        UUID4 string correlating this signal across logs, Langfuse, and OTel.
+    """
+
+    score: float
+    failure_type: FailureType
+    root_cause: str
+    hint: str
+    trace_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+
+    def __post_init__(self) -> None:
+        if not (0.0 <= self.score <= 1.0):
+            raise ValueError(
+                f"FailureSignal.score must be in [0.0, 1.0], got {self.score!r}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# FailurePattern
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class FailurePattern:
+    """Persistent failure pattern stored in a ``MemoryBackend``.
+
+    ``FailurePattern`` is the persisted form of a ``FailureSignal`` enriched
+    with an embedding and decay bookkeeping fields.  It represents a *recurring*
+    failure that the middleware has learned to anticipate and pre-empt.
+
+    Attributes
+    ----------
+    id : str
+        UUID4 string, unique per pattern.
+    query_embedding : List[float]
+        Dense embedding of the query that triggered this failure, produced by
+        the MiniLM encoder.  Used for cosine-similarity lookup in Phase 1.
+    failure_type : FailureType
+        Classified failure mode (mirrors ``FailureSignal.failure_type``).
+    root_cause : str
+        Human-readable root-cause explanation.
+    hint_text : str
+        Actionable hint to inject during Phase 1 augmentation.
+    score : float
+        Evaluation score at the time this pattern was last written/updated.
+    created_at : datetime
+        UTC timestamp of pattern creation.
+    decay_weight : float
+        Starts at ``1.0`` on creation.  Increases on reinforcement (+0.1),
+        decreases on non-improvement (−0.2).  Patterns below ``0.1`` are evicted.
+        See ``patternmem.decay`` for the exact update logic.
+    """
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    query_embedding: list[float] = field(default_factory=list)
+    failure_type: FailureType = FailureType.UNKNOWN
+    root_cause: str = ""
+    hint_text: str = ""
+    score: float = 0.0
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    decay_weight: float = 1.0
+
+    def __post_init__(self) -> None:
+        if not (0.0 <= self.score <= 1.0):
+            raise ValueError(
+                f"FailurePattern.score must be in [0.0, 1.0], got {self.score!r}"
+            )
+        if self.decay_weight < 0.0:
+            raise ValueError(
+                f"FailurePattern.decay_weight must be >= 0.0, got {self.decay_weight!r}"
+            )