sentinelrag 0.2.0__py3-none-any.whl

sentinelrag/__init__.py

@@ -0,0 +1,3 @@
+"""SentinelRAG — adaptive, attack-tested, eval-gated RAG."""
+
+__version__ = "0.2.0"

sentinelrag/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI application package."""

sentinelrag/api/app.py

@@ -0,0 +1,257 @@
+"""FastAPI entrypoint: health, ingest, and retrieval-backed query."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Literal
+
+from fastapi import FastAPI, Query
+from pydantic import BaseModel, Field
+
+from sentinelrag import __version__
+from sentinelrag.eval.phase2_rag import run_phase2_eval
+from sentinelrag.generation.llm import Message, get_llm
+from sentinelrag.ingest.pipeline import IndexState, ingest_corpus_dir, ingest_document_body
+from sentinelrag.paths import mini_corpus_dir
+from sentinelrag.retrieval.agentic import AgenticMultiHopRetriever
+from sentinelrag.retrieval.embeddings import build_embedder
+from sentinelrag.retrieval.reranker import build_reranker
+from sentinelrag.retrieval.stores import ScoredChunk
+from sentinelrag.retrieval.strategies import HybridRerankRetriever, NaiveRetriever, StrategyName
+from sentinelrag.router.policy import route_query
+from sentinelrag.settings import Settings, get_settings, settings_without_dotenv
+
+ApiStrategy = StrategyName | Literal["agentic", "auto"]
+
+HELLO_PASSAGE = {
+    "doc_id": "doc-hello-001",
+    "chunk_id": "chunk-hello-001",
+    "text": (
+        "SentinelRAG is an adaptive, attack-tested, eval-gated RAG platform. "
+        "Phase 0 proves a hello query path with MockLLM and citations."
+    ),
+}
+
+
+class HealthResponse(BaseModel):
+    status: str
+    version: str
+    env: str
+    llm_provider: str
+    index_ready: bool
+
+
+class IngestRequest(BaseModel):
+    corpus_path: str | None = Field(
+        default=None,
+        description="Filesystem path to a markdown corpus directory",
+    )
+    doc_id: str | None = None
+    title: str | None = None
+    text: str | None = None
+
+
+class IngestResponse(BaseModel):
+    chunks_indexed: int
+    corpus_path: str | None = None
+
+
+class QueryRequest(BaseModel):
+    query: str = Field(..., min_length=1, max_length=4000)
+
+
+class Citation(BaseModel):
+    doc_id: str
+    chunk_id: str
+    snippet: str
+
+
+class QueryResponse(BaseModel):
+    answer: str
+    citations: list[Citation]
+    strategy: str
+    provider: str
+
+
+class EvalRunResponse(BaseModel):
+    mean_faithfulness: float
+    mean_relevance: float
+    mean_recall_at_k: float
+    judge_prompt_version: str
+    n_golden_items: int
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    cfg = settings or get_settings()
+    app = FastAPI(title=cfg.app_name, version=__version__)
+    app.state.settings = cfg
+    app.state.index = IndexState()
+
+    def _maybe_bootstrap_index(current: Settings) -> None:
+        state: IndexState = app.state.index
+        if state.is_ready():
+            return
+        path = mini_corpus_dir()
+        if path.is_dir():
+            ingest_corpus_dir(path, state, settings=current)
+
+    @app.get("/health", response_model=HealthResponse)
+    def health() -> HealthResponse:
+        current = app.state.settings
+        assert isinstance(current, Settings)
+        state: IndexState = app.state.index
+        return HealthResponse(
+            status="ok",
+            version=__version__,
+            env=current.env,
+            llm_provider=current.llm_provider,
+            index_ready=state.is_ready(),
+        )
+
+    @app.post("/ingest", response_model=IngestResponse)
+    def ingest(body: IngestRequest) -> IngestResponse:
+        current = app.state.settings
+        assert isinstance(current, Settings)
+        state: IndexState = app.state.index
+        if body.text and body.doc_id:
+            count = ingest_document_body(
+                doc_id=body.doc_id,
+                title=body.title or body.doc_id,
+                text=body.text,
+                state=state,
+                settings=current,
+            )
+            return IngestResponse(chunks_indexed=count, corpus_path=None)
+        corpus = Path(body.corpus_path) if body.corpus_path else mini_corpus_dir()
+        count = ingest_corpus_dir(corpus, state, settings=current)
+        return IngestResponse(chunks_indexed=count, corpus_path=str(corpus))
+
+    @app.post("/query", response_model=QueryResponse)
+    def query_post(
+        body: QueryRequest,
+        strategy: Annotated[ApiStrategy, Query()] = "hybrid",
+    ) -> QueryResponse:
+        current = app.state.settings
+        assert isinstance(current, Settings)
+        _maybe_bootstrap_index(current)
+        state: IndexState = app.state.index
+        llm = get_llm(current)
+
+        if not state.is_ready() or state.store is None or state.embedder is None:
+            prompt = (
+                f"Context: {HELLO_PASSAGE['text']}\n\n"
+                f"Question: {body.query}\n\n"
+                "Answer using only the context."
+            )
+            answer = llm.complete([Message(role="user", content=prompt)])
+            return QueryResponse(
+                answer=answer,
+                citations=[
+                    Citation(
+                        doc_id=HELLO_PASSAGE["doc_id"],
+                        chunk_id=HELLO_PASSAGE["chunk_id"],
+                        snippet=HELLO_PASSAGE["text"][:160],
+                    )
+                ],
+                strategy="hello_stub",
+                provider=llm.name,
+            )
+
+        resolved = strategy
+        if strategy == "auto":
+            resolved = route_query(body.query, current).strategy
+
+        embedder = state.embedder or build_embedder(
+            current.embedding_model,
+            dim=current.embed_dim,
+            seed=current.benchmark_seed,
+        )
+        hits: list[ScoredChunk]
+        if resolved == "naive":
+            hits = NaiveRetriever(state.store, embedder).retrieve(
+                body.query, top_k=current.retrieval_top_k
+            )
+        elif resolved == "agentic":
+            hybrid = HybridRerankRetriever(
+                state.store,
+                state.bm25,
+                embedder,
+                build_reranker(current.reranker_model),
+                alpha=current.hybrid_alpha,
+                chunks_by_id=state.chunks,
+            )
+            agent = AgenticMultiHopRetriever(hybrid, max_hops=current.agent_max_hops)
+            hits = agent.retrieve(body.query, top_k=current.retrieval_top_k).chunks
+            resolved = "agentic"
+        else:
+            hits = HybridRerankRetriever(
+                state.store,
+                state.bm25,
+                embedder,
+                build_reranker(current.reranker_model),
+                alpha=current.hybrid_alpha,
+                chunks_by_id=state.chunks,
+            ).retrieve(body.query, top_k=current.retrieval_top_k)
+            resolved = "hybrid"
+        context = "\n\n".join(h.text for h in hits) if hits else HELLO_PASSAGE["text"]
+        prompt = f"Context:\n{context}\n\nQuestion: {body.query}\n\nAnswer using only the context."
+        answer = llm.complete([Message(role="user", content=prompt)])
+        citations = [
+            Citation(doc_id=h.doc_id, chunk_id=h.chunk_id, snippet=h.text[:160]) for h in hits
+        ]
+        if not citations:
+            citations = [
+                Citation(
+                    doc_id=HELLO_PASSAGE["doc_id"],
+                    chunk_id=HELLO_PASSAGE["chunk_id"],
+                    snippet=HELLO_PASSAGE["text"][:160],
+                )
+            ]
+        return QueryResponse(
+            answer=answer,
+            citations=citations,
+            strategy=resolved,
+            provider=llm.name,
+        )
+
+    @app.post("/eval/run", response_model=EvalRunResponse)
+    def eval_run() -> EvalRunResponse:
+        current = app.state.settings
+        assert isinstance(current, Settings)
+        offline = settings_without_dotenv(
+            llm_provider="mock",
+            embedding_model="mock-embedder",
+            reranker_model="mock-reranker",
+            vector_store_backend="memory",
+            env=current.env,
+        )
+        report = run_phase2_eval(settings=offline)
+        return EvalRunResponse(
+            mean_faithfulness=float(report["mean_faithfulness"]),
+            mean_relevance=float(report["mean_relevance"]),
+            mean_recall_at_k=float(report["mean_recall_at_k"]),
+            judge_prompt_version=str(report["judge_prompt_version"]),
+            n_golden_items=int(report["n_golden_items"]),
+        )
+
+    return app
+
+
+app = create_app()
+
+
+def main() -> None:
+    """CLI entry for `uv run sentinelrag` / uvicorn."""
+    import uvicorn
+
+    settings = get_settings()
+    uvicorn.run(
+        "sentinelrag.api.app:app",
+        host=settings.host,
+        port=settings.port,
+        reload=False,
+    )
+
+
+if __name__ == "__main__":
+    main()

sentinelrag/eval/__init__.py

@@ -0,0 +1 @@
+"""Golden datasets, metrics, and LLM-as-judge (Phase 2+)."""

sentinelrag/eval/agreement.py

@@ -0,0 +1,54 @@
+"""Judge vs human agreement on a calibration set."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from sentinelrag.eval.judge import Judge, MockJudge
+
+
+@dataclass(frozen=True, slots=True)
+class AgreementReport:
+    prompt_version: str
+    n_samples: int
+    mean_absolute_error: float
+    within_0_2: float
+
+
+def load_calibration(path: Path) -> tuple[str, list[dict[str, Any]]]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    version = str(payload.get("judge_prompt_version", "v1"))
+    samples = list(payload.get("samples", []))
+    return version, samples
+
+
+def judge_agreement(
+    path: Path,
+    *,
+    judge: Judge | None = None,
+) -> AgreementReport:
+    version, samples = load_calibration(path)
+    j = judge or MockJudge(prompt_version=version)
+    if not samples:
+        return AgreementReport(version, 0, 0.0, 0.0)
+    abs_errors: list[float] = []
+    within: list[float] = []
+    for sample in samples:
+        human = float(sample["human_faithfulness"])
+        scored = j.faithfulness(
+            context=str(sample["context"]),
+            question=str(sample["question"]),
+            answer=str(sample["answer"]),
+        )
+        err = abs(scored.score - human)
+        abs_errors.append(err)
+        within.append(1.0 if err <= 0.2 else 0.0)
+    return AgreementReport(
+        prompt_version=version,
+        n_samples=len(samples),
+        mean_absolute_error=sum(abs_errors) / len(abs_errors),
+        within_0_2=sum(within) / len(within),
+    )

sentinelrag/eval/golden_io.py

@@ -0,0 +1,38 @@
+"""Golden dataset JSONL interchange."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        rows.append(json.loads(line))
+    return rows
+
+
+def export_jsonl(items: list[dict[str, Any]], path: Path) -> None:
+    lines = [json.dumps(item, ensure_ascii=False) for item in items]
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def from_golden_dataset(dataset_path: Path, out_path: Path) -> None:
+    data = json.loads(dataset_path.read_text(encoding="utf-8"))
+    items = []
+    for row in data.get("items", []):
+        items.append(
+            {
+                "id": row["id"],
+                "query": row["query"],
+                "expected_chunk_ids": row.get("relevant_chunk_ids", []),
+                "kind": row.get("item_type", "single_hop"),
+                "citations_required": True,
+            }
+        )
+    export_jsonl(items, out_path)

sentinelrag/eval/judge.py

@@ -0,0 +1,111 @@
+"""LLM-as-judge with versioned prompts and MockJudge for CI."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+
+from sentinelrag.generation.llm import LLMProvider, Message, get_llm
+from sentinelrag.settings import Settings, get_settings
+
+_JUDGE_ROOT = Path(__file__).resolve().parent / "judges"
+
+
+@dataclass(frozen=True, slots=True)
+class JudgeScore:
+    score: float
+    reason: str
+    prompt_version: str
+
+
+class Judge(ABC):
+    @abstractmethod
+    def faithfulness(self, *, context: str, question: str, answer: str) -> JudgeScore:
+        """Score how well ``answer`` is supported by ``context``."""
+
+    @abstractmethod
+    def relevance(self, *, question: str, answer: str) -> JudgeScore:
+        """Score how well ``answer`` addresses ``question``."""
+
+
+def load_prompt(version: str, name: str) -> str:
+    path = _JUDGE_ROOT / version / f"{name}.txt"
+    if not path.is_file():
+        msg = f"Missing judge prompt: {path}"
+        raise FileNotFoundError(msg)
+    return path.read_text(encoding="utf-8")
+
+
+def _parse_judge_json(raw: str) -> tuple[float, str]:
+    match = re.search(r"\{.*\}", raw, flags=re.DOTALL)
+    payload = json.loads(match.group(0) if match else raw)
+    score = float(payload["score"])
+    reason = str(payload.get("reason", ""))
+    return max(0.0, min(1.0, score)), reason
+
+
+class MockJudge(Judge):
+    """Deterministic lexical judge for offline CI (not a human surrogate)."""
+
+    def __init__(self, *, prompt_version: str = "v1") -> None:
+        self.prompt_version = prompt_version
+
+    def faithfulness(self, *, context: str, question: str, answer: str) -> JudgeScore:
+        _ = question
+        ctx_tokens = set(context.lower().split())
+        ans_tokens = [t for t in answer.lower().split() if len(t) > 3]
+        if not ans_tokens:
+            return JudgeScore(0.0, "empty answer", self.prompt_version)
+        supported = sum(1 for t in ans_tokens if t in ctx_tokens)
+        score = supported / len(ans_tokens)
+        # Stable tie-break from content hash for reproducibility.
+        digest = hashlib.sha256(f"{context}:{answer}".encode()).hexdigest()
+        jitter = (int(digest[:4], 16) % 7) / 100.0
+        score = min(1.0, score * 0.85 + jitter)
+        return JudgeScore(round(score, 4), "mock lexical overlap", self.prompt_version)
+
+    def relevance(self, *, question: str, answer: str) -> JudgeScore:
+        q = set(question.lower().split())
+        a = set(answer.lower().split())
+        overlap = len(q & a) / max(len(q), 1)
+        score = min(1.0, overlap + 0.5)
+        return JudgeScore(round(score, 4), "mock question overlap", self.prompt_version)
+
+
+class LLMJudge(Judge):
+    """Uses configured LLM provider with versioned prompts (network in prod)."""
+
+    def __init__(
+        self,
+        *,
+        llm: LLMProvider | None = None,
+        prompt_version: str = "v1",
+        settings: Settings | None = None,
+    ) -> None:
+        self._llm = llm or get_llm(settings or get_settings())
+        self.prompt_version = prompt_version
+
+    def faithfulness(self, *, context: str, question: str, answer: str) -> JudgeScore:
+        template = load_prompt(self.prompt_version, "faithfulness")
+        prompt = template.format(context=context, question=question, answer=answer)
+        raw = self._llm.complete([Message(role="user", content=prompt)])
+        score, reason = _parse_judge_json(raw)
+        return JudgeScore(score, reason, self.prompt_version)
+
+    def relevance(self, *, question: str, answer: str) -> JudgeScore:
+        template = load_prompt(self.prompt_version, "relevance")
+        prompt = template.format(question=question, answer=answer)
+        raw = self._llm.complete([Message(role="user", content=prompt)])
+        score, reason = _parse_judge_json(raw)
+        return JudgeScore(score, reason, self.prompt_version)
+
+
+def get_judge(settings: Settings | None = None) -> Judge:
+    cfg = settings or get_settings()
+    if cfg.llm_provider == "mock":
+        return MockJudge()
+    return LLMJudge(settings=cfg)

sentinelrag/eval/judges/v1/faithfulness.txt

@@ -0,0 +1,17 @@
+You are evaluating whether an assistant answer is faithful to the provided context.
+
+Rules:
+- Score 1.0 only if every factual claim in the answer is supported by the context.
+- Score 0.0 if the answer contradicts the context or invents major facts not in the context.
+- Partial support → score between 0.0 and 1.0.
+
+Respond with JSON only: {"score": <number>, "reason": "<short explanation>"}
+
+Context:
+{context}
+
+Question:
+{question}
+
+Answer:
+{answer}

sentinelrag/eval/judges/v1/manifest.yaml

@@ -0,0 +1,8 @@
+version: v1
+created: "2026-11-08"
+prompts:
+  faithfulness: faithfulness.txt
+  relevance: relevance.txt
+description: >
+  Judges score 0.0-1.0 with a one-line rationale. JSON-only model output:
+  {"score": <float>, "reason": "<string>"}

sentinelrag/eval/judges/v1/relevance.txt

@@ -0,0 +1,13 @@
+You are evaluating whether an assistant answer addresses the user question.
+
+Rules:
+- Score 1.0 if the answer directly addresses the question.
+- Score 0.0 if it is off-topic or refuses without cause.
+
+Respond with JSON only: {"score": <number>, "reason": "<short explanation>"}
+
+Question:
+{question}
+
+Answer:
+{answer}

sentinelrag/eval/latency.py

@@ -0,0 +1,15 @@
+"""Lightweight latency helpers for gate evidence."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def timed_ms(fn: Callable[[], T]) -> tuple[T, float]:
+    t0 = time.perf_counter()
+    value = fn()
+    return value, (time.perf_counter() - t0) * 1000.0

sentinelrag/eval/metrics.py

@@ -0,0 +1,39 @@
+"""Retrieval and grounding evaluation metrics."""
+
+from __future__ import annotations
+
+import re
+
+
+def recall_at_k(retrieved_ids: list[str], relevant_ids: set[str], *, k: int) -> float:
+    if not relevant_ids:
+        return 0.0
+    top = set(retrieved_ids[:k])
+    return len(top & relevant_ids) / len(relevant_ids)
+
+
+def hit_rate_at_k(retrieved_ids: list[str], relevant_ids: set[str], *, k: int) -> float:
+    """1.0 if at least one relevant id appears in top-k, else 0.0."""
+    if not relevant_ids:
+        return 0.0
+    return 1.0 if set(retrieved_ids[:k]) & relevant_ids else 0.0
+
+
+def mrr(retrieved_ids: list[str], relevant_ids: set[str]) -> float:
+    for rank, cid in enumerate(retrieved_ids, start=1):
+        if cid in relevant_ids:
+            return 1.0 / rank
+    return 0.0
+
+
+_CITATION = re.compile(r"\[(?:cite|source|doc|chunk)[:\s]?([^\]]+)\]", re.IGNORECASE)
+
+
+def citation_coverage(answer: str, retrieved_ids: list[str]) -> float:
+    """Share of citation markers that point at retrieved chunk ids (0–1)."""
+    marks = _CITATION.findall(answer)
+    if not marks:
+        return 0.0
+    allowed = set(retrieved_ids)
+    hits = sum(1 for m in marks if m.strip() in allowed)
+    return hits / len(marks)

sentinelrag/eval/phase1_retrieval.py

@@ -0,0 +1,110 @@
+"""Phase 1 retrieval benchmark (library entry for scripts and tests)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from sentinelrag.eval.metrics import hit_rate_at_k, mrr, recall_at_k
+from sentinelrag.ingest.pipeline import IndexState, ingest_corpus_dir
+from sentinelrag.paths import mini_corpus_dir, mini_qrels_path
+from sentinelrag.retrieval.reranker import build_reranker
+from sentinelrag.retrieval.strategies import HybridRerankRetriever, NaiveRetriever, StrategyName
+from sentinelrag.settings import Settings, settings_without_dotenv
+
+
+def _default_benchmark_settings() -> Settings:
+    """Offline benchmark defaults (mock embedder; ignores ``.env`` file only)."""
+    return settings_without_dotenv(
+        embedding_model="mock-embedder",
+        reranker_model="mock-reranker",
+        vector_store_backend="memory",
+    )
+
+
+def corpus_content_hash(corpus_dir: Path) -> str:
+    digests: list[str] = []
+    for path in sorted(corpus_dir.glob("*.md")):
+        digests.append(hashlib.sha256(path.read_bytes()).hexdigest())
+    joined = "|".join(digests)
+    return hashlib.sha256(joined.encode()).hexdigest()[:16]
+
+
+def load_qrels(path: Path) -> tuple[int, list[dict[str, Any]]]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    seed = int(payload.get("seed", 42))
+    queries = list(payload.get("queries", []))
+    return seed, queries
+
+
+def run_benchmark(
+    *,
+    settings: Settings | None = None,
+    strategies: tuple[StrategyName, ...] = ("naive", "hybrid"),
+    corpus_dir: Path | None = None,
+    qrels_path: Path | None = None,
+) -> dict[str, Any]:
+    cfg = settings if settings is not None else _default_benchmark_settings()
+    corpus_dir = corpus_dir if corpus_dir is not None else mini_corpus_dir()
+    qrels_path = qrels_path if qrels_path is not None else mini_qrels_path()
+    seed, qrels = load_qrels(qrels_path)
+
+    state = IndexState()
+    ingest_corpus_dir(corpus_dir, state, settings=cfg)
+    assert state.store is not None and state.embedder is not None
+
+    embedder = state.embedder
+    k = cfg.retrieval_top_k
+    rows: list[dict[str, Any]] = []
+
+    for strategy in strategies:
+        retriever: NaiveRetriever | HybridRerankRetriever
+        if strategy == "naive":
+            retriever = NaiveRetriever(state.store, embedder)
+        else:
+            retriever = HybridRerankRetriever(
+                state.store,
+                state.bm25,
+                embedder,
+                build_reranker(cfg.reranker_model),
+                alpha=cfg.hybrid_alpha,
+                chunks_by_id=state.chunks,
+            )
+        recalls: list[float] = []
+        hits_rate: list[float] = []
+        mrrs: list[float] = []
+        latencies: list[float] = []
+        for item in qrels:
+            query = str(item["query"])
+            relevant = set(item["relevant_chunk_ids"])
+
+            t0 = time.perf_counter()
+            retrieved_ids = [h.chunk_id for h in retriever.retrieve(query, top_k=k)]
+            latencies.append((time.perf_counter() - t0) * 1000.0)
+            recalls.append(recall_at_k(retrieved_ids, relevant, k=k))
+            hits_rate.append(hit_rate_at_k(retrieved_ids, relevant, k=k))
+            mrrs.append(mrr(retrieved_ids, relevant))
+        rows.append(
+            {
+                "strategy": strategy,
+                "recall_at_k": sum(recalls) / len(recalls) if recalls else 0.0,
+                "hit_rate_at_k": sum(hits_rate) / len(hits_rate) if hits_rate else 0.0,
+                "mrr": sum(mrrs) / len(mrrs) if mrrs else 0.0,
+                "latency_ms_mean": sum(latencies) / len(latencies) if latencies else 0.0,
+            }
+        )
+
+    return {
+        "generated_at": datetime.now(tz=UTC).isoformat(),
+        "embedding_model": cfg.embedding_model,
+        "reranker_model": cfg.reranker_model,
+        "corpus_hash": corpus_content_hash(corpus_dir),
+        "seed": seed,
+        "retrieval_top_k": k,
+        "hybrid_alpha": cfg.hybrid_alpha,
+        "results": rows,
+    }