rerankgate 0.2.0__py3-none-any.whl

rerankgate/rewrite.py

@@ -0,0 +1,38 @@
+"""Query rewriting without an LLM: RM3-style pseudo-relevance feedback.
+
+Takes the top documents of a failed attempt and expands the query with
+high-value terms from them — the classic relevance-feedback trick,
+deterministic and free.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from rerankgate.retrieval import ScoredChunk
+from rerankgate.tokenize import STOPWORDS, tokenize
+
+
+class RM3Rewriter:
+    """Expand the query with discriminative terms from top failed-attempt docs."""
+
+    def __init__(self, expansion_terms: int = 5, top_docs: int = 3) -> None:
+        self.expansion_terms = expansion_terms
+        self.top_docs = top_docs
+
+    def rewrite(self, query: str, evidence: list[ScoredChunk]) -> str:
+        """Return an expanded query; returns the original if nothing to add."""
+        q_terms = set(tokenize(query))
+        term_counts: Counter[str] = Counter()
+        for sc in evidence[: self.top_docs]:
+            for t in tokenize(sc.chunk.text):
+                if t not in STOPWORDS and t not in q_terms and len(t) > 2:
+                    term_counts[t] += 1
+        if not term_counts:
+            return query
+        # Prefer terms that appear across multiple top docs (consensus terms).
+        ranked = sorted(term_counts.items(), key=lambda kv: (-kv[1], kv[0]))
+        new_terms = [t for t, _ in ranked[: self.expansion_terms]]
+        if not new_terms:
+            return query
+        return query.rstrip() + " " + " ".join(new_terms)

rerankgate/store.py

@@ -0,0 +1,78 @@
+"""Document store: JSONL persistence for chunks + optional numpy vectors."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import asdict
+
+from rerankgate.documents import Chunk
+from rerankgate.vectorstore import _HAS_NUMPY
+
+
+class DocumentStore:
+    """Persist chunks as JSONL; vectors as .npy when numpy is available."""
+
+    def __init__(self, path: str) -> None:
+        self.path = path
+
+    @property
+    def chunks_file(self) -> str:
+        return os.path.join(self.path, "chunks.jsonl")
+
+    @property
+    def vectors_file(self) -> str:
+        return os.path.join(self.path, "vectors.npy")
+
+    @property
+    def meta_file(self) -> str:
+        return os.path.join(self.path, "meta.json")
+
+    def save(
+        self,
+        chunks: list[Chunk],
+        vectors: list[list[float]] | None = None,
+        meta: dict | None = None,
+    ) -> None:
+        os.makedirs(self.path, exist_ok=True)
+        with open(self.chunks_file, "w", encoding="utf-8") as f:
+            for ch in chunks:
+                f.write(json.dumps(asdict(ch)) + "\n")
+        with open(self.meta_file, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "chunk_count": len(chunks),
+                    "has_vectors": vectors is not None,
+                    "dim": len(vectors[0]) if vectors else 0,
+                    **(meta or {}),
+                },
+                f,
+            )
+        if vectors is not None:
+            if not _HAS_NUMPY:
+                raise RuntimeError("numpy is required to persist vectors")
+            import numpy as np
+
+            np.save(self.vectors_file, np.asarray(vectors, dtype=np.float32))
+
+    def load(self) -> tuple[list[Chunk], list[list[float]] | None, dict]:
+        if not os.path.exists(self.chunks_file):
+            raise FileNotFoundError(f"no store at {self.path!r}: run `rerankgate ingest` first")
+        chunks: list[Chunk] = []
+        with open(self.chunks_file, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    chunks.append(Chunk(**json.loads(line)))
+        meta: dict = {}
+        if os.path.exists(self.meta_file):
+            with open(self.meta_file, encoding="utf-8") as f:
+                meta = json.load(f)
+        vectors: list[list[float]] | None = None
+        if meta.get("has_vectors") and os.path.exists(self.vectors_file):
+            if not _HAS_NUMPY:
+                raise RuntimeError("store has vectors but numpy is not installed")
+            import numpy as np
+
+            vectors = np.load(self.vectors_file).astype(float).tolist()
+        return chunks, vectors, meta

rerankgate/tokenize.py

@@ -0,0 +1,29 @@
+"""Shared tokenization and stopwords."""
+
+from __future__ import annotations
+
+import re
+
+_TOKEN = re.compile(r"[a-z0-9]+")
+
+STOPWORDS = frozenset(
+    """
+    a an the and or but if then else when while of at by for with about into through
+    during before after above below to from up down in out on off over under again
+    further once here there all any both each few more most other some such no nor
+    not only own same so than too very can will just don should now is are was were
+    be been being have has had having do does did doing would could ought i you he
+    she it we they them his her its our their this that these those am as it s t
+    what which who whom how why where when
+    """.split()
+)
+
+
+def tokenize(text: str) -> list[str]:
+    """Lowercase alphanumeric tokens."""
+    return _TOKEN.findall(text.lower())
+
+
+def content_tokens(text: str) -> list[str]:
+    """Tokens with stopwords removed."""
+    return [t for t in tokenize(text) if t not in STOPWORDS]

rerankgate/vectorstore.py

@@ -0,0 +1,66 @@
+"""Cosine-similarity vector store over chunk embeddings.
+
+Uses numpy when available; falls back to pure Python otherwise.
+Vectors are expected to be L2-normalized (cosine == dot product).
+"""
+
+from __future__ import annotations
+
+import math
+
+from rerankgate.documents import Chunk
+
+try:
+    import numpy as _np
+
+    _HAS_NUMPY = True
+except ImportError:  # pragma: no cover
+    _np = None  # type: ignore
+    _HAS_NUMPY = False
+
+
+class VectorStore:
+    def __init__(self, dim: int) -> None:
+        self.dim = dim
+        self._chunks: list[Chunk] = []
+        self._vectors: list[list[float]] = []
+
+    def __len__(self) -> int:
+        return len(self._chunks)
+
+    @property
+    def vectors(self) -> list[list[float]]:
+        """The stored (normalized) vectors, in chunk order."""
+        return [list(v) for v in self._vectors]
+
+    def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None:
+        if len(chunks) != len(vectors):
+            raise ValueError("chunks and vectors must have equal length")
+        for v in vectors:
+            if len(v) != self.dim:
+                raise ValueError(f"expected dim {self.dim}, got {len(v)}")
+        self._chunks.extend(chunks)
+        self._vectors.extend([list(map(float, v)) for v in vectors])
+
+    def search(self, query_vec: list[float], k: int = 100) -> list[tuple[Chunk, float]]:
+        """Top-k (chunk, cosine similarity) pairs, highest first."""
+        if not self._chunks:
+            return []
+        if _HAS_NUMPY:
+            mat = _np.asarray(self._vectors, dtype=_np.float64)
+            q = _np.asarray(query_vec, dtype=_np.float64)
+            qn = _np.linalg.norm(q) or 1.0
+            norms = _np.linalg.norm(mat, axis=1)
+            norms[norms == 0] = 1.0
+            sims = (mat @ q) / (norms * qn)
+            order = _np.argsort(-sims, kind="stable")[:k]
+            return [(self._chunks[i], float(sims[i])) for i in order]
+        # Pure-python fallback.
+        qn = math.sqrt(sum(x * x for x in query_vec)) or 1.0
+        scored = []
+        for ch, v in zip(self._chunks, self._vectors, strict=True):
+            vn = math.sqrt(sum(x * x for x in v)) or 1.0
+            sim = sum(a * b for a, b in zip(v, query_vec, strict=True)) / (vn * qn)
+            scored.append((ch, sim))
+        scored.sort(key=lambda p: p[1], reverse=True)
+        return scored[:k]

rerankgate-0.2.0.dist-info/METADATA

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: rerankgate
+Version: 0.2.0
+Summary: Agentic RAG quality gate: hybrid retrieval with a sufficiency gate, query-rewrite retries, and per-stage metrics. Local-first.
+Author: Calas Automations Inc.
+License: MIT
+Project-URL: Homepage, https://calasautomations.com
+Keywords: rag,retrieval,rerank,bm25,quality-gate,agentic
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: local
+Requires-Dist: numpy>=1.24; extra == "local"
+Requires-Dist: sentence-transformers>=2.2; extra == "local"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: ruff>=0.1; extra == "dev"
+
+# rerankgate
+
+**Agentic RAG quality gate.** A drop-in retrieval pipeline with a decision point
+between retrieval and generation:
+
+```
+query → hybrid retrieval (BM25 + vectors, RRF fusion)
+      → rerank
+      → sufficiency gate ── PASS ─→ evidence (+ optional generation & faithfulness check)
+                           FAIL ─→ rewrite query (RM3, no LLM) → retry (≤2, early-stop)
+                           exhausted → ABSTAIN with evidence + reasons
+```
+
+**The rule:** if the retrieval wasn't good enough, the answer never gets generated.
+Local-first — the sufficiency gate works with zero LLM calls by default.
+
+## Install
+
+```bash
+pip install rerankgate            # zero hard dependencies (BM25-only mode)
+pip install "rerankgate[local]"   # + sentence-transformers, numpy (hybrid vectors)
+```
+
+## Quickstart
+
+```python
+from rerankgate import RAGPipeline, Document
+
+pipe = RAGPipeline()  # degrades to BM25-only if embeddings aren't installed
+pipe.ingest([
+    Document(id="rag", text="Retrieval-augmented generation pairs a retriever..."),
+    Document(id="bm25", text="BM25 is a classic probabilistic ranking function..."),
+])
+
+result = pipe.ask("What is retrieval-augmented generation?")
+print(result.verdict)          # "answerable" | "insufficient" | "needs_review"
+print(result.gate.reasons)     # e.g. ["PASS"] or ["LOW_COVERAGE", ...]
+print(result.gate.explanation) # human-readable why
+
+# Suspected false negative? Override it — the override is recorded in the trace.
+result = pipe.ask("...", gate_override=True)
+
+# Optional: run your own generator through the faithfulness check.
+result = pipe.ask("...", generator=lambda q, ev: my_llm(q, [c.chunk.text for c in ev]))
+# verdict flips to "needs_review" if the answer isn't grounded in the evidence.
+```
+
+Every `ask()` returns a full `trace`: per-stage timings, retrieval modes,
+retry count, gate features, and whether the run was degraded.
+
+## CLI
+
+```bash
+# JSONL corpus: {"id": ..., "text": ..., ...metadata}
+rerankgate ingest --input corpus.jsonl --store ./store
+
+rerankgate query --store ./store "What is RAG?"
+rerankgate query --store ./store --gate-preset lenient --json "..."
+
+# Evaluate retrieval + gate on labeled data, then tune the threshold.
+# qrels JSONL: {"query": ..., "relevant_ids": [...], "answerable": true/false}
+rerankgate eval --store ./store --qrels eval.jsonl
+rerankgate eval --store ./store --qrels eval.jsonl --summary  # aggregates only
+```
+
+`rerankgate eval` reports recall@k, MRR (over answerable items), the gate
+confusion matrix (precision/recall of PASS vs your labels), and the abstention
+rate — use it to pick a threshold/preset instead of trusting the defaults.
+
+## How the gate works
+
+The sufficiency gate is a fixed weighted heuristic over interpretable features —
+top rerank score, query-term coverage, ranking margin, and cross-chunk support —
+plus hard floors on coverage and top score. Every decision carries stable
+machine-readable reason codes (`PASS`, `LOW_TOP_SCORE`, `LOW_COVERAGE`,
+`NO_MARGIN`, `NO_CONSENSUS`, `NO_EVIDENCE`) and a human explanation. Nothing is
+learned, so there's nothing to overfit; tune it with the eval CLI.
+
+Presets: `strict` (0.70) · `balanced` (0.55, default) · `lenient` (0.40).
+
+## Design notes
+
+- **Fusion:** Reciprocal Rank Fusion (k=60) — rank-based, no cross-system score
+  normalization hacks. Per-system weights available in `PipelineConfig`.
+- **Retries:** RM3-style pseudo-relevance feedback expands the query from the
+  failed attempt's top docs — no LLM needed. Hard cap of 2 retries plus
+  diminishing-returns early stop: the loop always terminates.
+- **Abstention > hallucination:** on `insufficient` the pipeline returns
+  best-effort evidence with gate reasons so the caller can decide — it never
+  silently hands weak context to an LLM.
+- **Pluggable:** `EmbeddingBackend`, `Reranker`, and `Judge` are protocols.
+  Bring your own embeddings (OpenAI, Cohere…), a cross-encoder, or an
+  LLM-as-judge without touching the pipeline.
+
+## Limitations (honest)
+
+- The heuristic gate is tuned for English keyword overlap. Synonym-heavy,
+  cross-lingual, or highly abstract queries can under-score — use hybrid vectors
+  and/or the `lenient` preset, and calibrate with `rerankgate eval`.
+- The default faithfulness judge is token-overlap based: free and instant, but
+  gameable by extractive copying. For adversarial settings, plug in `LLMJudge`.
+- Default chunking is a sentence-aware character window, not semantic chunking;
+  tables and structured text chunk naively.
+- `pip install rerankgate[local]` pulls PyTorch (~2GB) via sentence-transformers —
+  the price of local dense retrieval. BM25-only mode needs nothing.
+- The gate threshold defaults are starting points, not tuned constants — they
+  were sanity-checked on a toy corpus, not a benchmark. Calibrate on your data.
+- Not thread-safe; one pipeline per thread. At 500K+ vectors, prefer the numpy
+  path (installed with `[local]`) over the pure-Python fallback.
+
+## License
+
+MIT — Calas Automations Inc.

rerankgate-0.2.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+rerankgate/__init__.py,sha256=LzI5YMTl2bl3ik6MBC3FT9qZQLVYyrVePSpoFszpkOk,855
+rerankgate/bm25.py,sha256=TzRofy42skIxqDKPTJqR2vCoxVM2V6AyemGMtN2Gruk,2471
+rerankgate/cli.py,sha256=tswWUV1f8oRGsOM0L4LxZxnTVUU3HKTQsllZAze5H-g,4994
+rerankgate/config.py,sha256=UlvgtazRfiVs9EpmoxlVpEBiErDH_YN1aolLaVQLczo,2591
+rerankgate/documents.py,sha256=Eu-Bed4tZ2ztixuierwZu91ORwCV6_4PJhrwzzVHp3Q,2475
+rerankgate/embeddings.py,sha256=CafeivlHHPH8meKFCEIQUP5xO30P54FU9OHiLuPIPTQ,2411
+rerankgate/gate.py,sha256=xnc_f6-7bryHDb8XHFEkkS0Yz9Mnh8psNYnogf_FGAM,5955
+rerankgate/judge.py,sha256=DEm4ZtKYSSSgELfhDmDdfBQ_IWIyxIlz6bNt5f42BQc,3010
+rerankgate/metrics.py,sha256=eC4qj38Wcb_KgK7de4BBqGn1WxZt_K3Qq9sNZ1v5UJw,3755
+rerankgate/pipeline.py,sha256=34wFVV8-ghvU42FFQuZKDDYWqmNx2VvDt4rwQyeoM5E,11819
+rerankgate/rerank.py,sha256=93zekOAx-UKpMjUXn8iS7ebPZpDfs7aQLMAuFIoye-Y,3395
+rerankgate/retrieval.py,sha256=_CH1Ap4-H1XsIeCO2xXjocTVc50rmNhH5MqvH8oodbI,3450
+rerankgate/rewrite.py,sha256=HpIKS1CeaVxPipIDDawQI64ZBwwcNfKAiPGohMMu-_4,1498
+rerankgate/store.py,sha256=0vcpUP1WCusmg2OYEkc3hd8yYTJNNdNo-49TXt-pErA,2693
+rerankgate/tokenize.py,sha256=j0mALMp0SLSg2T7nt49hpiT_DGoUY-4aVmI02Suy9nI,984
+rerankgate/vectorstore.py,sha256=snOyoJ7NHq_BPsXHfweDNUhAjvEvUyj7NYvy7H3xJPs,2359
+rerankgate-0.2.0.dist-info/METADATA,sha256=4zXmMbtFCVNVQZxS3vKdsOTMG9aUTvZrkaY_Tiz1qYw,5912
+rerankgate-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rerankgate-0.2.0.dist-info/entry_points.txt,sha256=JGTMPUGp3iysG4ift10GXc4D9pmb9XCQ0wwVGoNBL6w,51
+rerankgate-0.2.0.dist-info/top_level.txt,sha256=P95LE_w0SpQ6j-91wwJwYu9bAEf-cYW6xR2gLCJ70IY,11
+rerankgate-0.2.0.dist-info/RECORD,,

rerankgate-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

rerankgate-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+rerankgate = rerankgate.cli:main

rerankgate-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+rerankgate