ragfallback 2.2.1__tar.gz → 2.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ragfallback-2.2.1 → ragfallback-2.2.3}/MANIFEST.in +0 -9
- {ragfallback-2.2.1/ragfallback.egg-info → ragfallback-2.2.3}/PKG-INFO +5 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/README.md +4 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/build_golden_dataset.py +6 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/chroma_real_kb_demo.py +18 -5
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/ci_regression_gate.py +4 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/financial_risk_analysis.py +11 -4
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/legal_document_analysis.py +21 -8
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/medical_research_synthesis.py +22 -9
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/real_data_demo.py +2 -5
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc10_metadata_sanitizer.py +17 -7
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc1_retrieval_health.py +4 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc2_embedding_guard.py +1 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc3_chunk_quality.py +10 -7
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc4_context_window.py +4 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc5_hybrid_failover.py +22 -8
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc6_adaptive_rag.py +9 -5
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc6_multi_hop_demo.py +10 -6
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc7_rag_evaluator.py +26 -11
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc8_context_stitcher.py +11 -5
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/uc9_embedding_probe.py +7 -8
- {ragfallback-2.2.1 → ragfallback-2.2.3}/pyproject.toml +1 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/__init__.py +1 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/core/__init__.py +0 -9
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/core/adaptive_retriever.py +108 -69
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/__init__.py +12 -3
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/chunking.py +4 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/context_window.py +1 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/embedding_probe.py +1 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/retrieval_health.py +6 -6
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/schema_sanitizer.py +3 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/stale_index.py +12 -4
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/evaluation/rag_evaluator.py +12 -6
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/__init__.py +4 -4
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/baseline_registry.py +11 -14
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/golden_runner.py +3 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/ragas_hook.py +5 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/retrieval/failover.py +8 -5
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/retrieval/smart_hybrid.py +4 -4
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/retrieval/wrappers.py +6 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/strategies/__init__.py +5 -10
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/strategies/base.py +6 -16
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/strategies/multi_hop.py +17 -11
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/strategies/query_variations.py +26 -20
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/tracking/__init__.py +7 -10
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/tracking/cache_monitor.py +4 -10
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/tracking/cost_tracker.py +26 -31
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/tracking/metrics.py +22 -20
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/utils/__init__.py +10 -11
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/utils/confidence_scorer.py +1 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/utils/embedding_factory.py +20 -40
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/utils/env.py +1 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/utils/llm_factory.py +40 -51
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/utils/vector_store_factory.py +30 -44
- {ragfallback-2.2.1 → ragfallback-2.2.3/ragfallback.egg-info}/PKG-INFO +5 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/requirements-dev.txt +0 -9
- {ragfallback-2.2.1 → ragfallback-2.2.3}/setup.py +2 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/__init__.py +0 -9
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/conftest.py +14 -14
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/integration/test_adaptive_workflow.py +26 -26
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/integration/test_chroma_pipeline.py +10 -3
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_adaptive_multi_hop_bridge.py +21 -9
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_async_retriever.py +7 -17
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_cache_monitor.py +16 -11
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_confidence_scorer.py +44 -28
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_cost_tracker.py +7 -17
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_diagnostics.py +13 -5
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_hybrid_retrieval.py +8 -10
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_metrics.py +12 -23
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_multi_hop.py +0 -1
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_query_variations.py +12 -17
- {ragfallback-2.2.1 → ragfallback-2.2.3}/INSTALL_AND_RUN.md +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/LICENSE +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/_kb_common.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/mlops_demo.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/production_reliability_example.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/examples/qdrant_local_demo.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/pytest.ini +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/context_stitcher.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/embedding_guard.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/diagnostics/embedding_validator.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/evaluation/__init__.py +2 -2
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/exceptions.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/locust_template.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/mlflow_logger.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/mlops/query_simulator.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/py.typed +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/retrieval/__init__.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback/retrieval/rerank_guard.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback.egg-info/SOURCES.txt +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback.egg-info/dependency_links.txt +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback.egg-info/requires.txt +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/ragfallback.egg-info/top_level.txt +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/setup.cfg +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/integration/__init__.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/__init__.py +0 -0
- {ragfallback-2.2.1 → ragfallback-2.2.3}/tests/unit/test_retrieval.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ragfallback
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.3
|
|
4
4
|
Summary: Prevents silent RAG failures — chunk quality, retrieval fallback, adaptive querying, and answer evaluation in one library.
|
|
5
5
|
Home-page: https://github.com/irfanalidv/ragfallback
|
|
6
6
|
Author: Irfan Ali
|
|
@@ -116,6 +116,7 @@ Drop into any LangChain-compatible stack. Catches bad chunks before they're embe
|
|
|
116
116
|
[](https://pypi.org/project/ragfallback/)
|
|
117
117
|
[](https://pepy.tech/project/ragfallback)
|
|
118
118
|
[](https://github.com/irfanalidv/ragfallback/actions/workflows/test.yml)
|
|
119
|
+
[](https://github.com/irfanalidv/ragfallback/actions/workflows/lint.yml)
|
|
119
120
|
[](https://pypi.org/project/ragfallback/)
|
|
120
121
|
[](https://github.com/irfanalidv/ragfallback/blob/main/LICENSE)
|
|
121
122
|
[](https://github.com/irfanalidv/ragfallback/stargazers)
|
|
@@ -595,7 +596,8 @@ RAGEvaluator (10 real Q&A pairs, heuristic, no LLM judge):
|
|
|
595
596
|
Avg overall : 62.9%
|
|
596
597
|
```
|
|
597
598
|
|
|
598
|
-
Install: `pip install ragfallback[chroma,huggingface,real-data]`
|
|
599
|
+
Install: `pip install ragfallback[chroma,huggingface,real-data]`
|
|
600
|
+
|
|
599
601
|
Dataset: [rajpurkar/squad](https://huggingface.co/datasets/rajpurkar/squad) — CC BY-SA 4.0
|
|
600
602
|
|
|
601
603
|
---
|
|
@@ -769,6 +771,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md). The quick version: run `pytest tests/uni
|
|
|
769
771
|
## License · Changelog
|
|
770
772
|
|
|
771
773
|
MIT License — see [LICENSE](LICENSE).
|
|
774
|
+
|
|
772
775
|
Full version history in [CHANGELOG.md](CHANGELOG.md).
|
|
773
776
|
|
|
774
777
|
---
|
|
@@ -9,6 +9,7 @@ Drop into any LangChain-compatible stack. Catches bad chunks before they're embe
|
|
|
9
9
|
[](https://pypi.org/project/ragfallback/)
|
|
10
10
|
[](https://pepy.tech/project/ragfallback)
|
|
11
11
|
[](https://github.com/irfanalidv/ragfallback/actions/workflows/test.yml)
|
|
12
|
+
[](https://github.com/irfanalidv/ragfallback/actions/workflows/lint.yml)
|
|
12
13
|
[](https://pypi.org/project/ragfallback/)
|
|
13
14
|
[](https://github.com/irfanalidv/ragfallback/blob/main/LICENSE)
|
|
14
15
|
[](https://github.com/irfanalidv/ragfallback/stargazers)
|
|
@@ -488,7 +489,8 @@ RAGEvaluator (10 real Q&A pairs, heuristic, no LLM judge):
|
|
|
488
489
|
Avg overall : 62.9%
|
|
489
490
|
```
|
|
490
491
|
|
|
491
|
-
Install: `pip install ragfallback[chroma,huggingface,real-data]`
|
|
492
|
+
Install: `pip install ragfallback[chroma,huggingface,real-data]`
|
|
493
|
+
|
|
492
494
|
Dataset: [rajpurkar/squad](https://huggingface.co/datasets/rajpurkar/squad) — CC BY-SA 4.0
|
|
493
495
|
|
|
494
496
|
---
|
|
@@ -662,6 +664,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md). The quick version: run `pytest tests/uni
|
|
|
662
664
|
## License · Changelog
|
|
663
665
|
|
|
664
666
|
MIT License — see [LICENSE](LICENSE).
|
|
667
|
+
|
|
665
668
|
Full version history in [CHANGELOG.md](CHANGELOG.md).
|
|
666
669
|
|
|
667
670
|
---
|
|
@@ -32,7 +32,9 @@ def _doc_id(text: str, prefix: str = "doc") -> str:
|
|
|
32
32
|
return f"{prefix}_{h}"
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def build_squad_samples(
|
|
35
|
+
def build_squad_samples(
|
|
36
|
+
n: int = 75,
|
|
37
|
+
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
|
36
38
|
"""
|
|
37
39
|
Load SQuAD validation split.
|
|
38
40
|
|
|
@@ -104,7 +106,9 @@ def build_squad_samples(n: int = 75) -> Tuple[List[Dict[str, Any]], List[Dict[st
|
|
|
104
106
|
return samples, docs_meta
|
|
105
107
|
|
|
106
108
|
|
|
107
|
-
def build_sciq_samples(
|
|
109
|
+
def build_sciq_samples(
|
|
110
|
+
n: int = 25,
|
|
111
|
+
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
|
108
112
|
"""
|
|
109
113
|
Load SciQ test split — science domain, harder than SQuAD.
|
|
110
114
|
|
|
@@ -33,6 +33,7 @@ warnings.filterwarnings(
|
|
|
33
33
|
)
|
|
34
34
|
|
|
35
35
|
import _kb_common
|
|
36
|
+
|
|
36
37
|
from ragfallback import AdaptiveRAGRetriever, CostTracker, MetricsCollector
|
|
37
38
|
from ragfallback.utils.llm_factory import create_open_source_llm
|
|
38
39
|
|
|
@@ -66,7 +67,9 @@ def _run_demo() -> int:
|
|
|
66
67
|
collection_name="ragfallback_kb_demo",
|
|
67
68
|
)
|
|
68
69
|
except ImportError as e:
|
|
69
|
-
print(
|
|
70
|
+
print(
|
|
71
|
+
f"{e}\nInstall: pip install chromadb sentence-transformers", file=sys.stderr
|
|
72
|
+
)
|
|
70
73
|
return 1
|
|
71
74
|
|
|
72
75
|
print("Chroma collection ready (embeddings computed from real file contents).\n")
|
|
@@ -118,7 +121,11 @@ def _run_demo() -> int:
|
|
|
118
121
|
)
|
|
119
122
|
except Exception as e:
|
|
120
123
|
err = str(e).lower()
|
|
121
|
-
if
|
|
124
|
+
if (
|
|
125
|
+
"connection refused" in err
|
|
126
|
+
or "11434" in err
|
|
127
|
+
or "failed to establish" in err
|
|
128
|
+
):
|
|
122
129
|
print(
|
|
123
130
|
"\n(Ollama not reachable — retrieval-only preview, no paid API keys.)\n"
|
|
124
131
|
"For full adaptive RAG: install https://ollama.ai and run: ollama pull llama3\n"
|
|
@@ -127,7 +134,9 @@ def _run_demo() -> int:
|
|
|
127
134
|
for i, doc in enumerate(hits, 1):
|
|
128
135
|
src = (doc.metadata or {}).get("source", "?")
|
|
129
136
|
body = (doc.page_content or "")[:400].replace("\n", " ")
|
|
130
|
-
print(
|
|
137
|
+
print(
|
|
138
|
+
f" [{i}] source={src}\n {body}{'…' if len(doc.page_content or '') > 400 else ''}\n"
|
|
139
|
+
)
|
|
131
140
|
return 0
|
|
132
141
|
print(
|
|
133
142
|
"Adaptive RAG failed. Is Ollama running?\n"
|
|
@@ -138,7 +147,9 @@ def _run_demo() -> int:
|
|
|
138
147
|
return 1
|
|
139
148
|
|
|
140
149
|
print(f"\nAnswer:\n {result.answer}\n")
|
|
141
|
-
print(
|
|
150
|
+
print(
|
|
151
|
+
f"Confidence: {result.confidence:.2%} | attempts: {result.attempts} | cost: ${result.cost:.4f}"
|
|
152
|
+
)
|
|
142
153
|
|
|
143
154
|
if result.intermediate_steps:
|
|
144
155
|
print("\nIntermediate steps (queries tried):")
|
|
@@ -153,7 +164,9 @@ def _run_demo() -> int:
|
|
|
153
164
|
def main() -> int:
|
|
154
165
|
import logging
|
|
155
166
|
|
|
156
|
-
logging.getLogger("ragfallback.strategies.query_variations").setLevel(
|
|
167
|
+
logging.getLogger("ragfallback.strategies.query_variations").setLevel(
|
|
168
|
+
logging.CRITICAL
|
|
169
|
+
)
|
|
157
170
|
with warnings.catch_warnings():
|
|
158
171
|
warnings.simplefilter("ignore")
|
|
159
172
|
return _run_demo()
|
|
@@ -158,14 +158,16 @@ async def run_gate() -> int:
|
|
|
158
158
|
print(
|
|
159
159
|
f" Comparing against baseline (recorded: {baseline.get('recorded_at', 'unknown')})"
|
|
160
160
|
)
|
|
161
|
-
print(
|
|
161
|
+
print(
|
|
162
|
+
" Threshold: 5% quality metrics; latency not gated (CI runners too noisy) → FAIL"
|
|
163
|
+
)
|
|
162
164
|
|
|
163
165
|
try:
|
|
164
166
|
registry.compare_or_fail(
|
|
165
167
|
report,
|
|
166
168
|
dataset=dataset_name,
|
|
167
169
|
threshold=0.05,
|
|
168
|
-
latency_threshold=5.0,
|
|
170
|
+
latency_threshold=5.0, # 500% — P95 latency varies wildly on GH Actions shared runners
|
|
169
171
|
)
|
|
170
172
|
registry.update(report, dataset=dataset_name)
|
|
171
173
|
print("\n RESULT: PASS ✓ — No regression detected")
|
|
@@ -13,11 +13,14 @@ Env vars : NONE required for retrieval demo; HF_TOKEN optional for LLM
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
-
import sys
|
|
17
16
|
import os
|
|
17
|
+
import sys
|
|
18
18
|
|
|
19
19
|
_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
20
|
-
if
|
|
20
|
+
if (
|
|
21
|
+
os.path.isdir(os.path.join(_repo_root, "ragfallback"))
|
|
22
|
+
and _repo_root not in sys.path
|
|
23
|
+
):
|
|
21
24
|
sys.path.insert(0, _repo_root)
|
|
22
25
|
|
|
23
26
|
_examples_dir = os.path.dirname(os.path.abspath(__file__))
|
|
@@ -66,7 +69,9 @@ def main() -> None:
|
|
|
66
69
|
|
|
67
70
|
checker = ChunkQualityChecker(min_chars=40)
|
|
68
71
|
report = checker.check(documents)
|
|
69
|
-
print(
|
|
72
|
+
print(
|
|
73
|
+
f"\nChunkQualityChecker: {report.n_chunks} sentences Violations: {len(report.violations)}"
|
|
74
|
+
)
|
|
70
75
|
|
|
71
76
|
import _kb_common
|
|
72
77
|
|
|
@@ -96,7 +101,9 @@ def main() -> None:
|
|
|
96
101
|
print(f" → {best}...")
|
|
97
102
|
|
|
98
103
|
print("\n✅ Financial RAG demo complete (no paid API keys used).")
|
|
99
|
-
print(
|
|
104
|
+
print(
|
|
105
|
+
" To add LLM generation: set HF_TOKEN env var and pass an LLM to AdaptiveRAGRetriever."
|
|
106
|
+
)
|
|
100
107
|
|
|
101
108
|
|
|
102
109
|
if __name__ == "__main__":
|
|
@@ -13,11 +13,14 @@ Env vars : NONE required
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
-
import sys
|
|
17
16
|
import os
|
|
17
|
+
import sys
|
|
18
18
|
|
|
19
19
|
_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
20
|
-
if
|
|
20
|
+
if (
|
|
21
|
+
os.path.isdir(os.path.join(_repo_root, "ragfallback"))
|
|
22
|
+
and _repo_root not in sys.path
|
|
23
|
+
):
|
|
21
24
|
sys.path.insert(0, _repo_root)
|
|
22
25
|
|
|
23
26
|
_examples_dir = os.path.dirname(os.path.abspath(__file__))
|
|
@@ -53,12 +56,20 @@ def main() -> None:
|
|
|
53
56
|
for row in ds:
|
|
54
57
|
ctx = (row.get("context") or "").strip()
|
|
55
58
|
if ctx and ctx not in seen and len(ctx) > 100:
|
|
56
|
-
documents.append(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
59
|
+
documents.append(
|
|
60
|
+
Document(
|
|
61
|
+
page_content=ctx[:600],
|
|
62
|
+
metadata={
|
|
63
|
+
"source": "cuad_contract",
|
|
64
|
+
"title": str(row.get("title", "")),
|
|
65
|
+
},
|
|
66
|
+
)
|
|
67
|
+
)
|
|
60
68
|
seen.add(ctx)
|
|
61
|
-
if
|
|
69
|
+
if (
|
|
70
|
+
row.get("question")
|
|
71
|
+
and test_question == "What are the termination conditions?"
|
|
72
|
+
):
|
|
62
73
|
test_question = row["question"]
|
|
63
74
|
if len(documents) >= 60:
|
|
64
75
|
break
|
|
@@ -75,7 +86,9 @@ def main() -> None:
|
|
|
75
86
|
|
|
76
87
|
checker = ChunkQualityChecker(min_chars=80)
|
|
77
88
|
report = checker.check(documents)
|
|
78
|
-
print(
|
|
89
|
+
print(
|
|
90
|
+
f"\nChunkQualityChecker: {report.n_chunks} clauses Violations: {len(report.violations)}"
|
|
91
|
+
)
|
|
79
92
|
|
|
80
93
|
import _kb_common
|
|
81
94
|
|
|
@@ -13,11 +13,14 @@ Env vars : NONE required
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
-
import sys
|
|
17
16
|
import os
|
|
17
|
+
import sys
|
|
18
18
|
|
|
19
19
|
_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
20
|
-
if
|
|
20
|
+
if (
|
|
21
|
+
os.path.isdir(os.path.join(_repo_root, "ragfallback"))
|
|
22
|
+
and _repo_root not in sys.path
|
|
23
|
+
):
|
|
21
24
|
sys.path.insert(0, _repo_root)
|
|
22
25
|
|
|
23
26
|
_examples_dir = os.path.dirname(os.path.abspath(__file__))
|
|
@@ -53,10 +56,12 @@ def main() -> None:
|
|
|
53
56
|
ctx_list = (row.get("context") or {}).get("contexts") or []
|
|
54
57
|
for ctx in ctx_list:
|
|
55
58
|
if ctx and len(ctx) > 100:
|
|
56
|
-
documents.append(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
59
|
+
documents.append(
|
|
60
|
+
Document(
|
|
61
|
+
page_content=ctx[:600],
|
|
62
|
+
metadata={"source": "pubmed"},
|
|
63
|
+
)
|
|
64
|
+
)
|
|
60
65
|
if row.get("question"):
|
|
61
66
|
test_questions.append(row["question"])
|
|
62
67
|
if len(documents) >= 60:
|
|
@@ -66,7 +71,9 @@ def main() -> None:
|
|
|
66
71
|
print("SKIP: No usable abstract segments loaded from dataset")
|
|
67
72
|
sys.exit(0)
|
|
68
73
|
|
|
69
|
-
test_question =
|
|
74
|
+
test_question = (
|
|
75
|
+
test_questions[0] if test_questions else "What are the treatment outcomes?"
|
|
76
|
+
)
|
|
70
77
|
|
|
71
78
|
print(f" Loaded {len(documents)} real PubMed abstract segments")
|
|
72
79
|
print(f" Example: {documents[0].page_content[:100]}...")
|
|
@@ -76,7 +83,9 @@ def main() -> None:
|
|
|
76
83
|
|
|
77
84
|
checker = ChunkQualityChecker(min_chars=80)
|
|
78
85
|
report = checker.check(documents)
|
|
79
|
-
print(
|
|
86
|
+
print(
|
|
87
|
+
f"\nChunkQualityChecker: {report.n_chunks} segments Violations: {len(report.violations)}"
|
|
88
|
+
)
|
|
80
89
|
|
|
81
90
|
import _kb_common
|
|
82
91
|
|
|
@@ -93,7 +102,11 @@ def main() -> None:
|
|
|
93
102
|
print(f" Indexed {len(documents)} real abstract segments")
|
|
94
103
|
|
|
95
104
|
print("\nRetrieval demo (no LLM required)...")
|
|
96
|
-
questions = [
|
|
105
|
+
questions = [
|
|
106
|
+
test_question,
|
|
107
|
+
"What are the side effects?",
|
|
108
|
+
"What does the study conclude?",
|
|
109
|
+
]
|
|
97
110
|
retriever = vs.as_retriever(search_kwargs={"k": 3})
|
|
98
111
|
for q in questions[:2]:
|
|
99
112
|
hits = retriever.invoke(q)
|
|
@@ -16,8 +16,8 @@ Env vars : NONE required
|
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
-
import sys
|
|
20
19
|
import os
|
|
20
|
+
import sys
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
23
|
# Allow running directly from a repository clone without pip install -e .
|
|
@@ -86,10 +86,7 @@ def main() -> None:
|
|
|
86
86
|
from ragfallback.diagnostics import RetrievalHealthCheck
|
|
87
87
|
|
|
88
88
|
# SQuAD answers are spans within context, so substring match is the correct check
|
|
89
|
-
probe_dict = {
|
|
90
|
-
p["question"]: p["ground_truth"][:60]
|
|
91
|
-
for p in probes[:20]
|
|
92
|
-
}
|
|
89
|
+
probe_dict = {p["question"]: p["ground_truth"][:60] for p in probes[:20]}
|
|
93
90
|
health_checker = RetrievalHealthCheck(k=4)
|
|
94
91
|
health_report = health_checker.run_substring_probes(
|
|
95
92
|
store,
|
|
@@ -26,8 +26,9 @@ _examples_dir = Path(__file__).resolve().parent
|
|
|
26
26
|
if str(_examples_dir) not in sys.path:
|
|
27
27
|
sys.path.insert(0, str(_examples_dir))
|
|
28
28
|
|
|
29
|
-
from langchain_core.documents import Document # noqa: E402
|
|
30
29
|
import _kb_common # noqa: E402
|
|
30
|
+
from langchain_core.documents import Document # noqa: E402
|
|
31
|
+
|
|
31
32
|
from ragfallback.diagnostics import sanitize_documents # noqa: E402
|
|
32
33
|
|
|
33
34
|
|
|
@@ -47,7 +48,11 @@ def main() -> None:
|
|
|
47
48
|
Document(
|
|
48
49
|
page_content="Python lists are mutable ordered sequences of elements.",
|
|
49
50
|
metadata={
|
|
50
|
-
"tags": [
|
|
51
|
+
"tags": [
|
|
52
|
+
"python",
|
|
53
|
+
"data-structures",
|
|
54
|
+
"lists",
|
|
55
|
+
], # list — rejected by Chroma
|
|
51
56
|
"score": 0.95,
|
|
52
57
|
"source": "python_guide.txt",
|
|
53
58
|
},
|
|
@@ -55,15 +60,18 @@ def main() -> None:
|
|
|
55
60
|
Document(
|
|
56
61
|
page_content="Dictionaries map hashable keys to arbitrary values.",
|
|
57
62
|
metadata={
|
|
58
|
-
"nested": {
|
|
59
|
-
|
|
63
|
+
"nested": {
|
|
64
|
+
"author": "Jane Doe",
|
|
65
|
+
"year": 2023,
|
|
66
|
+
}, # nested dict — rejected
|
|
67
|
+
"raw_bytes": b"binary content here", # bytes — rejected
|
|
60
68
|
"source": "python_guide.txt",
|
|
61
69
|
},
|
|
62
70
|
),
|
|
63
71
|
Document(
|
|
64
72
|
page_content="Functions are defined with the def keyword and return values.",
|
|
65
73
|
metadata={
|
|
66
|
-
"page": None,
|
|
74
|
+
"page": None, # None — handled inconsistently across stores
|
|
67
75
|
"reviewed": True,
|
|
68
76
|
"source": "python_guide.txt",
|
|
69
77
|
},
|
|
@@ -98,10 +106,12 @@ def main() -> None:
|
|
|
98
106
|
)
|
|
99
107
|
results = vs_clean.similarity_search("What are Python lists?", k=2)
|
|
100
108
|
print(f" Inserted {len(clean_docs)} docs. Retrieved {len(results)} result(s). ✓")
|
|
101
|
-
print(
|
|
109
|
+
print(
|
|
110
|
+
f"\n Metadata sanitized: {len(clean_docs)} document(s) ready for any vector store."
|
|
111
|
+
)
|
|
102
112
|
|
|
103
113
|
print("\nKey transformations applied by sanitize_documents():")
|
|
104
|
-
print(
|
|
114
|
+
print(' list / tuple → JSON string (e.g. \'["python", "lists"]\')')
|
|
105
115
|
print(" dict → flattened keys with dot notation (nested.key → value)")
|
|
106
116
|
print(" bytes → UTF-8 decoded string")
|
|
107
117
|
print(" None → preserved as None (most stores handle scalar None)")
|
|
@@ -15,13 +15,16 @@ import tempfile
|
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
|
|
17
17
|
import _kb_common
|
|
18
|
+
|
|
18
19
|
from ragfallback.diagnostics import RetrievalHealthCheck
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
def main() -> None:
|
|
22
23
|
docs = _kb_common.load_sample_kb()
|
|
23
24
|
persist = Path(tempfile.mkdtemp(prefix="ragfallback_uc1_")) / "chroma"
|
|
24
|
-
vs = _kb_common.build_chroma_store(
|
|
25
|
+
vs = _kb_common.build_chroma_store(
|
|
26
|
+
docs, persist_directory=persist, collection_name="uc1"
|
|
27
|
+
)
|
|
25
28
|
health = RetrievalHealthCheck(k=4)
|
|
26
29
|
report = health.quick_check(vs, docs, sample_size=min(10, len(docs)), seed=42)
|
|
27
30
|
print(report.summary())
|
|
@@ -30,6 +30,7 @@ if str(_examples_dir) not in sys.path:
|
|
|
30
30
|
sys.path.insert(0, str(_examples_dir))
|
|
31
31
|
|
|
32
32
|
import _kb_common # noqa: E402
|
|
33
|
+
|
|
33
34
|
from ragfallback.diagnostics import ChunkQualityChecker # noqa: E402
|
|
34
35
|
|
|
35
36
|
|
|
@@ -67,9 +68,9 @@ def main() -> None:
|
|
|
67
68
|
|
|
68
69
|
print("\nRunning ChunkQualityChecker with production thresholds...")
|
|
69
70
|
checker = ChunkQualityChecker(
|
|
70
|
-
min_chars=80,
|
|
71
|
-
max_chars=4000,
|
|
72
|
-
min_words=8,
|
|
71
|
+
min_chars=80, # flag anything shorter than a sentence
|
|
72
|
+
max_chars=4000, # flag unusually long passages
|
|
73
|
+
min_words=8, # flag stubs
|
|
73
74
|
)
|
|
74
75
|
report = checker.check(texts)
|
|
75
76
|
print(report.summary())
|
|
@@ -96,9 +97,9 @@ def main() -> None:
|
|
|
96
97
|
# Inject a synthetic bad chunk to demonstrate the detection path
|
|
97
98
|
print("\n Injecting 3 synthetic bad chunks to demonstrate detection...")
|
|
98
99
|
synthetic_bad = texts + [
|
|
99
|
-
"Too short.",
|
|
100
|
-
"incomplete sentence that just cuts",
|
|
101
|
-
texts[0][:50] + " " + texts[0][:50],
|
|
100
|
+
"Too short.", # 10 chars
|
|
101
|
+
"incomplete sentence that just cuts", # 34 chars, no period
|
|
102
|
+
texts[0][:50] + " " + texts[0][:50], # near-duplicate
|
|
102
103
|
]
|
|
103
104
|
bad_report = checker.check(synthetic_bad)
|
|
104
105
|
print(f" After injection: {bad_report.summary()}")
|
|
@@ -128,7 +129,9 @@ def main() -> None:
|
|
|
128
129
|
print(" Re-check: 0 violations after auto_fix ✓")
|
|
129
130
|
else:
|
|
130
131
|
remaining = len(fixed_report.violations)
|
|
131
|
-
print(
|
|
132
|
+
print(
|
|
133
|
+
f" Re-check: {remaining} violation(s) remain (auto_fix is conservative by design)"
|
|
134
|
+
)
|
|
132
135
|
|
|
133
136
|
print("\n✅ UC-3 demo complete.")
|
|
134
137
|
print(" Dataset: SQuAD (Wikipedia) — CC BY-SA 4.0")
|
|
@@ -15,13 +15,16 @@ import tempfile
|
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
|
|
17
17
|
import _kb_common
|
|
18
|
+
|
|
18
19
|
from ragfallback.diagnostics import ContextWindowGuard
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
def main() -> None:
|
|
22
23
|
docs = _kb_common.load_sample_kb()
|
|
23
24
|
persist = Path(tempfile.mkdtemp(prefix="ragfallback_uc4_")) / "chroma"
|
|
24
|
-
vs = _kb_common.build_chroma_store(
|
|
25
|
+
vs = _kb_common.build_chroma_store(
|
|
26
|
+
docs, persist_directory=persist, collection_name="uc4"
|
|
27
|
+
)
|
|
25
28
|
emb = _kb_common.get_embeddings()
|
|
26
29
|
q = "refund policy authentication"
|
|
27
30
|
retrieved = vs.similarity_search(q, k=12)
|
|
@@ -25,7 +25,11 @@ if (_repo_root / "ragfallback").is_dir() and str(_repo_root) not in sys.path:
|
|
|
25
25
|
sys.path.insert(0, str(_repo_root))
|
|
26
26
|
|
|
27
27
|
import _kb_common # noqa: E402
|
|
28
|
-
|
|
28
|
+
|
|
29
|
+
from ragfallback.retrieval import ( # noqa: E402
|
|
30
|
+
FailoverRetriever,
|
|
31
|
+
SmartThresholdHybridRetriever,
|
|
32
|
+
)
|
|
29
33
|
|
|
30
34
|
|
|
31
35
|
def demo_hybrid_bm25_fallback(docs, vs) -> None:
|
|
@@ -56,7 +60,9 @@ def demo_hybrid_bm25_fallback(docs, vs) -> None:
|
|
|
56
60
|
|
|
57
61
|
# _dense_scores_are_weak is part of the public API — demonstrate it
|
|
58
62
|
print(f"\n _dense_scores_are_weak([]) → {hybrid._dense_scores_are_weak([])}")
|
|
59
|
-
print(
|
|
63
|
+
print(
|
|
64
|
+
f" _dense_scores_are_weak([doc]) → {hybrid._dense_scores_are_weak(docs_out[:1])}"
|
|
65
|
+
)
|
|
60
66
|
|
|
61
67
|
|
|
62
68
|
def demo_failover_on_empty(vs) -> None:
|
|
@@ -65,6 +71,7 @@ def demo_failover_on_empty(vs) -> None:
|
|
|
65
71
|
|
|
66
72
|
class EmptyRetriever:
|
|
67
73
|
"""Simulates a retriever that always returns nothing (e.g. index gap)."""
|
|
74
|
+
|
|
68
75
|
def invoke(self, query: str):
|
|
69
76
|
return []
|
|
70
77
|
|
|
@@ -76,8 +83,10 @@ def demo_failover_on_empty(vs) -> None:
|
|
|
76
83
|
docs_out = fb.invoke(query)
|
|
77
84
|
print(f"Results: {len(docs_out)} doc(s) from fallback")
|
|
78
85
|
for d in docs_out:
|
|
79
|
-
print(
|
|
80
|
-
|
|
86
|
+
print(
|
|
87
|
+
f" source={d.metadata.get('ragfallback_retrieval_source')!r} "
|
|
88
|
+
f"text='{d.page_content[:60].replace(chr(10), ' ')}…'"
|
|
89
|
+
)
|
|
81
90
|
|
|
82
91
|
|
|
83
92
|
def demo_failover_on_exception(vs) -> None:
|
|
@@ -86,6 +95,7 @@ def demo_failover_on_exception(vs) -> None:
|
|
|
86
95
|
|
|
87
96
|
class BrokenRetriever:
|
|
88
97
|
"""Simulates a downed remote vector store."""
|
|
98
|
+
|
|
89
99
|
def invoke(self, query: str):
|
|
90
100
|
raise ConnectionError("Pinecone endpoint unreachable")
|
|
91
101
|
|
|
@@ -97,8 +107,10 @@ def demo_failover_on_exception(vs) -> None:
|
|
|
97
107
|
docs_out = fb.invoke(query)
|
|
98
108
|
print(f"Results: {len(docs_out)} doc(s) from fallback")
|
|
99
109
|
for d in docs_out:
|
|
100
|
-
print(
|
|
101
|
-
|
|
110
|
+
print(
|
|
111
|
+
f" source={d.metadata.get('ragfallback_retrieval_source')!r} "
|
|
112
|
+
f"text='{d.page_content[:60].replace(chr(10), ' ')}…'"
|
|
113
|
+
)
|
|
102
114
|
|
|
103
115
|
|
|
104
116
|
def demo_failover_min_results(vs) -> None:
|
|
@@ -116,8 +128,10 @@ def demo_failover_min_results(vs) -> None:
|
|
|
116
128
|
docs_out = fb.invoke(query)
|
|
117
129
|
print(f"Results: {len(docs_out)} doc(s) (fallback returned more)")
|
|
118
130
|
for d in docs_out:
|
|
119
|
-
print(
|
|
120
|
-
|
|
131
|
+
print(
|
|
132
|
+
f" source={d.metadata.get('ragfallback_retrieval_source')!r} "
|
|
133
|
+
f"text='{d.page_content[:60].replace(chr(10), ' ')}…'"
|
|
134
|
+
)
|
|
121
135
|
|
|
122
136
|
|
|
123
137
|
def main() -> None:
|
|
@@ -31,11 +31,11 @@ if str(_examples_dir) not in sys.path:
|
|
|
31
31
|
sys.path.insert(0, str(_examples_dir))
|
|
32
32
|
|
|
33
33
|
import _kb_common # noqa: E402
|
|
34
|
+
|
|
34
35
|
from ragfallback import AdaptiveRAGRetriever # noqa: E402
|
|
35
36
|
from ragfallback.strategies import QueryVariationsStrategy # noqa: E402
|
|
36
37
|
from ragfallback.tracking import CostTracker, MetricsCollector # noqa: E402
|
|
37
38
|
|
|
38
|
-
|
|
39
39
|
# Pre-scripted variations for three demo queries. The mock cycles through
|
|
40
40
|
# them deterministically so the retry chain is always visible even offline.
|
|
41
41
|
_MOCK_VARIATIONS: dict[int, list[str]] = {
|
|
@@ -167,7 +167,9 @@ def main() -> None:
|
|
|
167
167
|
print(f" → attempt {attempt}: '{query}...' confidence={conf:.0%}")
|
|
168
168
|
|
|
169
169
|
print(f" Final answer : {(result.answer or 'Not found')[:120]}")
|
|
170
|
-
print(
|
|
170
|
+
print(
|
|
171
|
+
f" Succeeded at : attempt {result.attempts} confidence={result.confidence:.0%}"
|
|
172
|
+
)
|
|
171
173
|
if result.cost == 0.0:
|
|
172
174
|
print(f" Cost : $0.0000 (mock LLM — no real inference)")
|
|
173
175
|
else:
|
|
@@ -175,9 +177,11 @@ def main() -> None:
|
|
|
175
177
|
|
|
176
178
|
stats = adaptive.metrics_collector.get_stats()
|
|
177
179
|
print("\n" + "─" * 66)
|
|
178
|
-
print(
|
|
179
|
-
|
|
180
|
-
|
|
180
|
+
print(
|
|
181
|
+
f"Session stats: {stats['total_queries']} queries "
|
|
182
|
+
f"success_rate={stats['success_rate']:.0%} "
|
|
183
|
+
f"avg_confidence={stats['avg_confidence']:.0%}"
|
|
184
|
+
)
|
|
181
185
|
|
|
182
186
|
if llm.__class__.__name__ == "MagicMock":
|
|
183
187
|
print(
|