ragfallback 2.1.0__tar.gz → 2.2.0__tar.gz

{ragfallback-2.1.0/ragfallback.egg-info → ragfallback-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragfallback
-Version: 2.1.0
+Version: 2.2.0
 Summary: Prevents silent RAG failures — chunk quality, retrieval fallback, adaptive querying, and answer evaluation in one library.
 Home-page: https://github.com/irfanalidv/ragfallback
 Author: Irfan Ali

{ragfallback-2.1.0 → ragfallback-2.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ragfallback"
-version = "2.1.0"
+version = "2.2.0"
 description = "Prevents silent RAG failures — chunk quality, retrieval fallback, adaptive querying, and answer evaluation in one library."
 readme = "README.md"
 requires-python = ">=3.8"

{ragfallback-2.1.0 → ragfallback-2.2.0}/ragfallback/__init__.py

@@ -9,16 +9,18 @@ This module exposes a small curated shortcut only (see ``__all__``).
 
 from __future__ import annotations
 
-__version__ = "2.1.0"
+__version__ = "2.2.0"
 __author__ = "Irfan Ali"
 
 from ragfallback.core.adaptive_retriever import AdaptiveRAGRetriever, QueryResult
+from ragfallback.tracking.cache_monitor import CacheMonitor
 from ragfallback.tracking.cost_tracker import CostTracker
 from ragfallback.tracking.metrics import MetricsCollector
 
 __all__ = [
     "AdaptiveRAGRetriever",
     "QueryResult",
+    "CacheMonitor",
     "CostTracker",
     "MetricsCollector",
 ]

{ragfallback-2.1.0 → ragfallback-2.2.0}/ragfallback/core/adaptive_retriever.py

@@ -1,11 +1,15 @@
 """Retriever wrapper that retries failed or low-confidence queries using pluggable strategies."""
 
-from typing import List, Optional, Dict, Any, Tuple
-from dataclasses import dataclass
-import logging
-import time
+from __future__ import annotations
+
+import asyncio
+import functools
 import json
+import logging
 import re
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
 
 from langchain_core.vectorstores import VectorStore
 from langchain_core.language_models import BaseLanguageModel
@@ -270,6 +274,279 @@ Return your answer in JSON format: {"answer": "...", "source": "..."}"""
             intermediate_steps=intermediate_steps if return_intermediate_steps else None
         )
 
+    async def aquery_with_fallback(
+        self,
+        question: str,
+        context: Optional[Dict[str, Any]] = None,
+        return_intermediate_steps: bool = False,
+        enforce_budget: bool = False,
+    ) -> QueryResult:
+        """Async mirror of :meth:`query_with_fallback` using LangChain ``ainvoke``.
+
+        Uses ``ainvoke`` on the retriever and LLM so concurrent callers can overlap
+        I/O-bound work. Falls back transparently to a thread-pool for objects that
+        do not implement ``ainvoke``.
+
+        Args:
+            question: The question to answer.
+            context: Optional metadata filters / context dict.
+            return_intermediate_steps: Include all attempt dicts in the result.
+            enforce_budget: Stop early if the cost budget is exceeded.
+
+        Returns:
+            :class:`QueryResult` with the same fields as the sync version.
+        """
+        context = context or {}
+        intermediate_steps: List[Dict[str, Any]] = []
+        total_cost = 0.0
+        loop = asyncio.get_event_loop()
+        start = loop.time()
+
+        for strategy_idx, strategy in enumerate(self.strategies):
+            if strategy_idx >= self.max_attempts:
+                break
+
+            if enforce_budget and self.cost_tracker.budget_exceeded():
+                if self.logger:
+                    self.logger.warning("Budget exceeded, stopping fallback attempts")
+                break
+
+            attempt_num = strategy_idx + 1
+
+            # Delegate multi-hop strategies to thread pool (they have no async API)
+            if callable(getattr(strategy, "run", None)):
+                if self.logger:
+                    self.logger.debug(
+                        "strategy %s has run() — delegating to thread pool",
+                        strategy.__class__.__name__,
+                    )
+                retriever = self.vector_store.as_retriever()
+                hop_result = await loop.run_in_executor(
+                    None,
+                    functools.partial(
+                        strategy.run,
+                        question=question,
+                        retriever=retriever,
+                        llm=self.llm,
+                    ),
+                )
+                step_data: Dict[str, Any] = {
+                    "attempt": attempt_num,
+                    "query": question,
+                    "strategy": "multi_hop",
+                    "hops": hop_result.total_hops,
+                    "answer": hop_result.final_answer,
+                    "confidence": 0.85 if hop_result.success else 0.0,
+                    "cost": 0.0,
+                }
+                intermediate_steps.append(step_data)
+                if hop_result.success and hop_result.final_answer:
+                    latency_ms = (loop.time() - start) * 1000
+                    self.metrics_collector.record_success(
+                        attempts=attempt_num,
+                        confidence=0.85,
+                        cost=total_cost,
+                        latency_ms=latency_ms,
+                        strategy_used="multi_hop",
+                    )
+                    return QueryResult(
+                        answer=hop_result.final_answer,
+                        source="multi_hop",
+                        confidence=0.85,
+                        attempts=attempt_num,
+                        cost=total_cost,
+                        intermediate_steps=intermediate_steps if return_intermediate_steps else None,
+                    )
+                continue
+
+            queries = strategy.generate_queries(
+                original_query=question,
+                context=context,
+                attempt=strategy_idx + 1,
+                llm=self.llm,
+            )
+
+            for query_idx, query in enumerate(queries):
+                attempt_num = strategy_idx * len(queries) + query_idx + 1
+
+                if attempt_num > self.max_attempts:
+                    break
+
+                if self.logger:
+                    self.logger.info(
+                        "async attempt %d/%d: %s", attempt_num, self.max_attempts, query[:100]
+                    )
+
+                docs = await self._aretrieve_documents(query, context)
+
+                if not docs:
+                    if self.logger:
+                        self.logger.warning("no documents found for query: %s", query)
+                    intermediate_steps.append(
+                        {"attempt": attempt_num, "query": query,
+                         "documents": 0, "confidence": 0.0, "cost": 0.0}
+                    )
+                    continue
+
+                answer, source, confidence, cost = await self._agenerate_answer(
+                    question=question,
+                    query=query,
+                    documents=docs,
+                    context=context,
+                )
+
+                total_cost += cost
+                latency_ms = (loop.time() - start) * 1000
+
+                step_data = {
+                    "attempt": attempt_num,
+                    "query": query,
+                    "documents": len(docs),
+                    "answer": answer,
+                    "source": source,
+                    "confidence": confidence,
+                    "cost": cost,
+                }
+                intermediate_steps.append(step_data)
+
+                if confidence >= self.min_confidence and answer.lower() not in [
+                    "x", "not found", "n/a", "unknown"
+                ]:
+                    if self.logger:
+                        self.logger.debug(
+                            "async attempt %d succeeded (confidence %.2f)",
+                            attempt_num,
+                            confidence,
+                        )
+                    self.metrics_collector.record_success(
+                        attempts=attempt_num,
+                        confidence=confidence,
+                        cost=total_cost,
+                        latency_ms=latency_ms,
+                        strategy_used=strategy.get_name(),
+                    )
+                    return QueryResult(
+                        answer=answer,
+                        source=source,
+                        confidence=confidence,
+                        attempts=attempt_num,
+                        cost=total_cost,
+                        intermediate_steps=intermediate_steps if return_intermediate_steps else None,
+                    )
+
+        latency_ms = (loop.time() - start) * 1000
+
+        if self.logger:
+            self.logger.warning(
+                "all %d async attempts exhausted without meeting confidence threshold",
+                len(intermediate_steps),
+            )
+
+        if intermediate_steps:
+            best_attempt = max(intermediate_steps, key=lambda x: x.get("confidence", 0.0))
+            best_answer = best_attempt.get("answer", "No answer found")
+            best_source = best_attempt.get("source", "")
+            best_confidence = best_attempt.get("confidence", 0.0)
+        else:
+            best_answer = "No answer found"
+            best_source = ""
+            best_confidence = 0.0
+
+        self.metrics_collector.record_failure(
+            attempts=len(intermediate_steps),
+            cost=total_cost,
+            latency_ms=latency_ms,
+            strategy_used=self.strategies[0].get_name() if self.strategies else "unknown",
+        )
+
+        return QueryResult(
+            answer=best_answer,
+            source=best_source,
+            confidence=best_confidence,
+            attempts=len(intermediate_steps) or 1,
+            cost=total_cost,
+            intermediate_steps=intermediate_steps if return_intermediate_steps else None,
+        )
+
+    async def _aretrieve_documents(
+        self, query: str, context: Dict[str, Any]
+    ) -> List[Any]:
+        """Async document retrieval; falls back to thread pool if ``ainvoke`` absent."""
+        try:
+            search_kwargs = self._build_search_kwargs(context)
+            retriever = self.vector_store.as_retriever(search_kwargs=search_kwargs)
+            ainvoke = getattr(retriever, "ainvoke", None)
+            if ainvoke is not None:
+                result = await ainvoke(query)
+                return list(result or [])
+            # Fall back: run sync invoke in executor
+            loop = asyncio.get_event_loop()
+            invoke = getattr(retriever, "invoke", retriever.get_relevant_documents)
+            return list(
+                await loop.run_in_executor(None, functools.partial(invoke, query)) or []
+            )
+        except Exception as exc:
+            if self.logger:
+                self.logger.error("async retrieve error: %s", exc)
+            return []
+
+    async def _agenerate_answer(
+        self,
+        question: str,
+        query: str,
+        documents: List[Any],
+        context: Dict[str, Any],
+    ) -> Tuple[str, str, float, float]:
+        """Async answer generation; falls back to thread pool if ``ainvoke`` absent."""
+        docs_text = self._format_documents(documents)
+        prompt = self._build_answer_prompt(question, docs_text, context)
+        loop = asyncio.get_event_loop()
+
+        with self.cost_tracker.track(operation="answer_generation"):
+            messages = [
+                SystemMessage(content=self.answer_prompt_template),
+                HumanMessage(content=prompt),
+            ]
+            try:
+                ainvoke = getattr(self.llm, "ainvoke", None)
+                if ainvoke is not None:
+                    response = await ainvoke(messages)
+                else:
+                    response = await loop.run_in_executor(
+                        None, functools.partial(self.llm.invoke, messages)
+                    )
+            except AttributeError:
+                response = await loop.run_in_executor(
+                    None, functools.partial(self.llm.invoke, messages)
+                )
+
+            answer_text = response.content if hasattr(response, "content") else str(response)
+
+            if hasattr(response, "response_metadata"):
+                metadata = response.response_metadata
+                if "token_usage" in metadata:
+                    usage = metadata["token_usage"]
+                    self.cost_tracker.record_tokens(
+                        input_tokens=usage.get("prompt_tokens", 0),
+                        output_tokens=usage.get("completion_tokens", 0),
+                        model=getattr(self.llm, "model_name", "gpt-4"),
+                    )
+
+        answer, source = self._parse_answer(answer_text)
+        scorer = ConfidenceScorer(llm=self.llm)
+        confidence = await loop.run_in_executor(
+            None,
+            functools.partial(
+                scorer.score,
+                question=question,
+                answer=answer,
+                documents=documents,
+                context=context,
+            ),
+        )
+        cost = self.cost_tracker.get_last_cost()
+        return answer, source, confidence, cost
+
     def _retrieve_documents(
         self,
         query: str,

{ragfallback-2.1.0 → ragfallback-2.2.0}/ragfallback/mlops/golden_runner.py

@@ -3,12 +3,14 @@
 from __future__ import annotations
 
 import asyncio
+import functools
 import json
+import logging
 import time
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Sequence, Set, Union
+from typing import Any, Dict, List, Optional, Sequence, Set, Union
 
 import numpy as np
 
@@ -16,6 +18,8 @@ from ragfallback.core.adaptive_retriever import AdaptiveRAGRetriever, QueryResul
 from ragfallback.evaluation import recall_at_k
 from ragfallback.mlops.ragas_hook import RagasHook, RagasReport
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class GoldenReport:
@@ -30,6 +34,7 @@ class GoldenReport:
     n_samples: int
     timestamp: datetime
     per_sample: List[Dict[str, Any]]
+    cache_stats: Optional[Dict[str, Any]] = field(default=None)
 
 
 class GoldenRunner:
@@ -40,10 +45,22 @@ class GoldenRunner:
         retriever: AdaptiveRAGRetriever,
         ragas_hook: RagasHook,
         dataset: Union[str, List[Dict[str, Any]]],
+        cache_monitor: Optional[Any] = None,
     ) -> None:
-        """Load JSON path or use in-memory list of ``query`` / ``ground_truth`` / optional ids."""
+        """Load JSON path or use in-memory list; optionally wrap retriever with CacheMonitor.
+
+        Args:
+            retriever: :class:`~ragfallback.core.adaptive_retriever.AdaptiveRAGRetriever`
+                instance to query.
+            ragas_hook: :class:`~ragfallback.mlops.ragas_hook.RagasHook` for scoring.
+            dataset: JSON file path or list of ``{"query", "ground_truth", ...}`` dicts.
+            cache_monitor: Optional :class:`~ragfallback.tracking.cache_monitor.CacheMonitor`
+                instance. When provided, the retriever's vector store retriever is wrapped
+                to track cache hits/misses. Stats appear in ``GoldenReport.cache_stats``.
+        """
         self.retriever = retriever
         self.ragas_hook = ragas_hook
+        self._cache_monitor = cache_monitor
         if isinstance(dataset, str):
             raw = Path(dataset).read_text(encoding="utf-8")
             self._dataset = json.loads(raw)
@@ -57,11 +74,27 @@ class GoldenRunner:
     def _retrieve_docs(self, query: str, k: int = 5) -> List[Any]:
         """Fetch top-``k`` documents for context and id extraction."""
         r = self.retriever.vector_store.as_retriever(search_kwargs={"k": k})
+        if self._cache_monitor is not None:
+            r = self._cache_monitor.wrap_retriever(r, k=k)
         invoke = getattr(r, "invoke", None)
         if invoke is not None:
             return list(invoke(query) or [])
         return list(r.get_relevant_documents(query))
 
+    async def _aretrieve_docs(self, query: str, k: int = 5) -> List[Any]:
+        """Async fetch of top-``k`` documents."""
+        r = self.retriever.vector_store.as_retriever(search_kwargs={"k": k})
+        if self._cache_monitor is not None:
+            r = self._cache_monitor.wrap_retriever(r, k=k)
+        ainvoke = getattr(r, "ainvoke", None)
+        if ainvoke is not None:
+            return list(await ainvoke(query) or [])
+        loop = asyncio.get_event_loop()
+        invoke = getattr(r, "invoke", None)
+        if invoke is not None:
+            return list(await loop.run_in_executor(None, functools.partial(invoke, query)) or [])
+        return []
+
     def _doc_ids(self, docs: Sequence[Any]) -> List[str]:
         """Stable string ids from document metadata or content hash."""
         out: List[str] = []
@@ -77,9 +110,7 @@ class GoldenRunner:
 
     def _contexts_from_docs(self, docs: Sequence[Any]) -> List[str]:
         """Plain-text contexts for evaluation."""
-        return [
-            (getattr(d, "page_content", str(d)) or "") for d in docs
-        ]
+        return [(getattr(d, "page_content", str(d)) or "") for d in docs]
 
     def _ids_from_intermediate(self, result: QueryResult) -> List[str]:
         """Best-effort doc id list from intermediate steps (often empty)."""
@@ -92,7 +123,7 @@ class GoldenRunner:
         return []
 
     def _run_single(self, item: Dict[str, Any]) -> Dict[str, Any]:
-        """Run one golden row and return diagnostics plus ragas-oriented fields."""
+        """Run one golden row synchronously and return diagnostics."""
         query = item["query"]
         gt = item.get("ground_truth", "")
         rel_ids: Set[str] = set(str(x) for x in item.get("relevant_doc_ids", []))
@@ -111,7 +142,6 @@ class GoldenRunner:
 
         r3 = recall_at_k(retrieved_ids, rel_ids, 3)
         r5 = recall_at_k(retrieved_ids, rel_ids, 5)
-        fallback_triggered = result.attempts > 1
 
         return {
             "question": query,
@@ -121,7 +151,50 @@ class GoldenRunner:
             "contexts": contexts,
             "latency_ms": latency_ms,
             "retrieved_ids": retrieved_ids,
-            "fallback_triggered": fallback_triggered,
+            "fallback_triggered": result.attempts > 1,
+            "recall_at_3": r3,
+            "recall_at_5": r5,
+        }
+
+    async def _arun_single(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """Run one golden row natively async using :meth:`aquery_with_fallback`."""
+        query = item["query"]
+        gt = item.get("ground_truth", "")
+        rel_ids: Set[str] = set(str(x) for x in item.get("relevant_doc_ids", []))
+
+        try:
+            t0 = time.perf_counter()
+            result = await self.retriever.aquery_with_fallback(
+                query, return_intermediate_steps=True
+            )
+            latency_ms = (time.perf_counter() - t0) * 1000.0
+        except AttributeError:
+            logger.warning(
+                "retriever does not support aquery_with_fallback — "
+                "falling back to thread pool for query: %s",
+                query[:80],
+            )
+            loop = asyncio.get_event_loop()
+            return await loop.run_in_executor(None, self._run_single, item)
+
+        docs = await self._aretrieve_docs(query, k=5)
+        retrieved_ids = self._ids_from_intermediate(result)
+        if not retrieved_ids:
+            retrieved_ids = self._doc_ids(docs)
+        contexts = self._contexts_from_docs(docs)
+
+        r3 = recall_at_k(retrieved_ids, rel_ids, 3)
+        r5 = recall_at_k(retrieved_ids, rel_ids, 5)
+
+        return {
+            "question": query,
+            "ground_truth": gt,
+            "answer": result.answer,
+            "confidence": result.confidence,
+            "contexts": contexts,
+            "latency_ms": latency_ms,
+            "retrieved_ids": retrieved_ids,
+            "fallback_triggered": result.attempts > 1,
             "recall_at_3": r3,
             "recall_at_5": r5,
         }
@@ -139,6 +212,11 @@ class GoldenRunner:
         fb = sum(1 for s in per_sample if s.get("fallback_triggered")) / n
         p95 = float(np.percentile(latencies, 95)) if latencies else 0.0
         mean_lat = sum(latencies) / len(latencies) if latencies else 0.0
+        cache_stats = (
+            self._cache_monitor.get_stats().as_dict()
+            if self._cache_monitor is not None
+            else None
+        )
         return GoldenReport(
             ragas=ragas_report,
             recall_at_3=mean_r3,
@@ -149,6 +227,7 @@ class GoldenRunner:
             n_samples=len(per_sample),
             timestamp=datetime.utcnow(),
             per_sample=per_sample,
+            cache_stats=cache_stats,
         )
 
     def run(self) -> GoldenReport:
@@ -169,12 +248,8 @@ class GoldenRunner:
         return self._build_report(per_sample, ragas_rep)
 
     async def run_async(self) -> GoldenReport:
-        """Evaluate golden rows concurrently (thread pool per row), then Ragas async."""
-        loop = asyncio.get_event_loop()
-        tasks = [
-            loop.run_in_executor(None, self._run_single, item)
-            for item in self._dataset
-        ]
+        """Evaluate golden rows concurrently using native async, then Ragas async."""
+        tasks = [self._arun_single(item) for item in self._dataset]
         per_sample = list(await asyncio.gather(*tasks))
         ragas_samples = [
             {

ragfallback-2.2.0/ragfallback/tracking/__init__.py

@@ -0,0 +1,16 @@
+"""Cost tracking, metrics collection, and cache monitoring."""
+
+from ragfallback.tracking.cache_monitor import CacheMonitor, CacheStats
+from ragfallback.tracking.cost_tracker import CostTracker, ModelPricing
+from ragfallback.tracking.metrics import MetricsCollector
+
+__all__ = ["CacheMonitor", "CacheStats", "CostTracker", "ModelPricing", "MetricsCollector"]
+
+
+
+
+
+
+
+
+