cgs-rag 0.1.0__tar.gz

cgs_rag-0.1.0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: cgs-rag
+Version: 0.1.0
+Summary: Composite Grounding Score: multi-signal hallucination detection for production RAG systems
+Author-email: Nishant Kumar <nishant.k@marmeto.com>
+License: MIT
+Project-URL: Homepage, https://github.com/nishant-k-marmeto/cgs-rag
+Project-URL: Repository, https://github.com/nishant-k-marmeto/cgs-rag
+Keywords: rag,retrieval-augmented-generation,hallucination,detection,nlp,llm,grounding,faithfulness
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Requires-Dist: torch>=2.0.0
+Requires-Dist: transformers>=4.36.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pandas>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: jupyter; extra == "dev"
+Requires-Dist: matplotlib; extra == "dev"
+Requires-Dist: seaborn; extra == "dev"

cgs_rag-0.1.0/cgs_rag/__init__.py

@@ -0,0 +1,41 @@
+"""
+cgs-rag
+=======
+Composite Grounding Score — multi-signal hallucination detection for
+production RAG systems.
+
+Based on the thesis:
+  "Composite Grounding Score (CGS): A Multi-Signal Framework for
+   Hallucination Detection in Production RAG Systems"
+  Nishant Kumar, IIT Patna Executive M.Tech AI & DSE, 2025
+
+Quick start
+-----------
+    from cgs_rag import CGSDetector
+
+    detector = CGSDetector()
+    result = detector.score(
+        question = "What is the capital of France?",
+        answer   = "Berlin",
+        context  = "France is a country in Western Europe. Its capital is Paris."
+    )
+    print(result.risk_score)          # e.g. 0.782
+    print(result.is_hallucination)    # True
+    print(result.explain())
+
+Domain calibration
+------------------
+    import pandas as pd
+    val_df = pd.read_csv("my_labelled_rag_data.csv")
+    # columns needed: question, answer, context, label (1=hallucinated, 0=grounded)
+
+    detector.calibrate(val_df)
+    # Automatically detects signal direction, optimises weights & threshold.
+"""
+
+from .detector import CGSDetector
+from .result   import CGSResult
+
+__version__ = "0.1.0"
+__author__  = "Nishant Kumar"
+__all__     = ["CGSDetector", "CGSResult"]

cgs_rag-0.1.0/cgs_rag/calibration.py

@@ -0,0 +1,172 @@
+"""
+CGS Calibration Utilities
+=========================
+
+Three independent steps:
+  1. detect_signal_direction  — determine whether high cosine = risk or grounding
+  2. find_optimal_threshold   — grid search over τ to maximise F1
+  3. optimise_weights         — grid search over (α, β, γ) to maximise AUC
+
+All functions operate on numpy arrays and are independent of the detector class,
+making them easy to use standalone or in custom pipelines.
+"""
+
+from typing import Optional, Tuple
+import numpy as np
+from sklearn.metrics import roc_auc_score, f1_score
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+def detect_signal_direction(
+    s3_scores: np.ndarray,
+    labels:    np.ndarray,
+) -> str:
+    """
+    Automatically detect the cosine signal direction on a labelled sample.
+
+    Parameters
+    ----------
+    s3_scores : array of cosine similarities (one per sample)
+    labels    : binary array (1 = hallucinated, 0 = grounded)
+
+    Returns
+    -------
+    "cosine_as_risk"
+        E[s3 | hallucinated] > E[s3 | grounded]
+        → high cosine means the answer is *suspiciously* close to context
+        → e.g. HaluEval adversarial benchmark construction
+
+    "cosine_as_grounding"
+        E[s3 | grounded] >= E[s3 | hallucinated]
+        → high cosine means the answer is well-supported by context
+        → e.g. real RAG pipelines, TruthfulQA, PubMedQA
+    """
+    labels    = np.asarray(labels).astype(int)
+    s3_scores = np.asarray(s3_scores, dtype=float)
+
+    mean_hal = s3_scores[labels == 1].mean() if (labels == 1).any() else 0.5
+    mean_gnd = s3_scores[labels == 0].mean() if (labels == 0).any() else 0.5
+
+    if mean_hal > mean_gnd:
+        return "cosine_as_risk"
+    return "cosine_as_grounding"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+def find_optimal_threshold(
+    cgs_risk:     np.ndarray,
+    labels:       np.ndarray,
+    n_thresholds: int = 80,
+) -> Tuple[float, float]:
+    """
+    Grid search for the decision threshold τ that maximises F1 on the
+    provided labelled data.
+
+    Parameters
+    ----------
+    cgs_risk     : array of CGS risk scores
+    labels       : binary array (1 = hallucinated, 0 = grounded)
+    n_thresholds : number of candidate thresholds to evaluate
+
+    Returns
+    -------
+    (best_tau, best_f1)
+    """
+    labels   = np.asarray(labels).astype(int)
+    cgs_risk = np.asarray(cgs_risk, dtype=float)
+
+    best_f1, best_tau = 0.0, 0.40
+    for tau in np.linspace(0.10, 0.90, n_thresholds):
+        preds = (cgs_risk >= tau).astype(int)
+        f1    = f1_score(labels, preds, zero_division=0)
+        if f1 > best_f1:
+            best_f1  = f1
+            best_tau = float(tau)
+
+    return best_tau, best_f1
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+def optimise_weights(
+    s2_scores: np.ndarray,
+    s3_scores: np.ndarray,
+    labels:    np.ndarray,
+    direction: str,
+    s1_scores: Optional[np.ndarray] = None,
+    step:      float = 0.05,
+) -> Tuple[dict, float, float]:
+    """
+    Grid search over (α, β, γ) weights — constrained to α + β + γ = 1 —
+    to maximise AUC-ROC.
+
+    Parameters
+    ----------
+    s2_scores  : NLI entailment scores (Signal 2)
+    s3_scores  : cosine similarity scores (Signal 3)
+    labels     : binary array (1 = hallucinated, 0 = grounded)
+    direction  : "cosine_as_risk" or "cosine_as_grounding"
+    s1_scores  : token log-prob confidence (Signal 1); None = Lite mode
+    step       : grid step size (smaller = finer search, slower)
+
+    Returns
+    -------
+    (weights_dict, optimal_threshold, best_auc)
+    where weights_dict = {"alpha": ..., "beta": ..., "gamma": ...}
+    """
+    labels    = np.asarray(labels).astype(int)
+    s2_scores = np.asarray(s2_scores, dtype=float)
+    s3_scores = np.asarray(s3_scores, dtype=float)
+
+    # Apply signal direction to s3
+    s3_risk = s3_scores if direction == "cosine_as_risk" else (1.0 - s3_scores)
+
+    best_auc     = 0.0
+    best_weights = {"alpha": 0.0, "beta": 0.15, "gamma": 0.85}
+    best_tau     = 0.40
+
+    grid = np.arange(0.0, 1.0 + step / 2, step)
+
+    if s1_scores is not None:
+        s1_scores = np.asarray(s1_scores, dtype=float)
+        for alpha in grid:
+            for beta in grid:
+                gamma = 1.0 - alpha - beta
+                if gamma < -0.001 or gamma > 1.001:
+                    continue
+                gamma = max(0.0, min(1.0, gamma))
+                risk = alpha * (1.0 - s1_scores) + beta * (1.0 - s2_scores) + gamma * s3_risk
+                try:
+                    auc = roc_auc_score(labels, risk)
+                except ValueError:
+                    continue
+                if auc > best_auc:
+                    best_auc = auc
+                    tau, _   = find_optimal_threshold(risk, labels)
+                    best_tau = tau
+                    best_weights = {
+                        "alpha": round(float(alpha), 3),
+                        "beta":  round(float(beta), 3),
+                        "gamma": round(float(gamma), 3),
+                    }
+    else:
+        # Lite mode: α fixed at 0, search over β + γ = 1
+        for beta in grid:
+            gamma = 1.0 - beta
+            if gamma < 0:
+                continue
+            risk = beta * (1.0 - s2_scores) + gamma * s3_risk
+            try:
+                auc = roc_auc_score(labels, risk)
+            except ValueError:
+                continue
+            if auc > best_auc:
+                best_auc = auc
+                tau, _   = find_optimal_threshold(risk, labels)
+                best_tau = tau
+                best_weights = {
+                    "alpha": 0.0,
+                    "beta":  round(float(beta), 3),
+                    "gamma": round(float(gamma), 3),
+                }
+
+    return best_weights, best_tau, round(best_auc, 4)

cgs_rag-0.1.0/cgs_rag/detector.py

@@ -0,0 +1,301 @@
+"""
+CGSDetector — the main public class.
+
+Quick start
+-----------
+    from cgs_rag import CGSDetector
+
+    detector = CGSDetector()                        # auto-detect mode
+    result   = detector.score(q, answer, context)  # single response
+    results  = detector.score_batch(qs, ans, ctxs) # batch
+    detector.calibrate(val_df)                      # adapt to your domain
+
+Modes
+-----
+"auto"  Uses S1+S2+S3 (Full) if Ollama is reachable, else S2+S3 (Lite).
+"lite"  S2 (NLI) + S3 (cosine).  No LLM required.  Recommended default.
+"full"  S1 (token log-prob) + S2 + S3.  Requires local Ollama instance.
+
+Direction
+---------
+"cosine_as_risk"     : high cosine → hallucinated (HaluEval adversarial regime)
+"cosine_as_grounding": high cosine → grounded     (real RAG / TruthfulQA)
+
+If you don't call calibrate(), the default direction is "cosine_as_grounding"
+(the natural RAG regime), which is correct for most real-world deployments.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from typing import List, Optional, Union
+
+from .result import CGSResult
+from .signals.signal1_token import TokenLogProbSignal
+from .signals.signal2_nli   import NLISignal
+from .signals.signal3_cosine import CosineSignal
+from .calibration import (
+    detect_signal_direction,
+    find_optimal_threshold,
+    optimise_weights,
+)
+
+# ── Thesis-validated defaults (HaluEval test set) ────────────────────────────
+_DEFAULT_WEIGHTS    = {"alpha": 0.0, "beta": 0.15, "gamma": 0.85}
+_DEFAULT_THRESHOLD  = 0.40
+_DEFAULT_DIRECTION  = "cosine_as_grounding"   # safe default for real RAG
+
+
+class CGSDetector:
+    """
+    Composite Grounding Score hallucination detector.
+
+    Parameters
+    ----------
+    mode         : "auto" | "lite" | "full"
+    nli_model    : HuggingFace model ID for Signal 2
+    cosine_model : SentenceTransformer model ID for Signal 3
+    ollama_model : Ollama model name for Signal 1
+    ollama_url   : URL of the local Ollama service
+    threshold    : decision threshold τ (overridden by calibrate())
+    weights      : {"alpha", "beta", "gamma"} weight dict
+    direction    : "cosine_as_risk" or "cosine_as_grounding"
+    """
+
+    def __init__(
+        self,
+        mode:         str   = "auto",
+        nli_model:    str   = "cross-encoder/nli-deberta-v3-small",
+        cosine_model: str   = "all-MiniLM-L6-v2",
+        ollama_model: str   = "llama3.2",
+        ollama_url:   str   = "http://localhost:11434",
+        threshold:    float = _DEFAULT_THRESHOLD,
+        weights:      Optional[dict] = None,
+        direction:    str   = _DEFAULT_DIRECTION,
+    ):
+        self.mode      = mode
+        self.threshold = threshold
+        self.weights   = dict(weights) if weights else dict(_DEFAULT_WEIGHTS)
+        self.direction = direction
+        self._calibrated = False
+
+        # Signals are lazy-loaded on first use
+        self._s1 = TokenLogProbSignal(model_name=ollama_model, ollama_url=ollama_url)
+        self._s2 = NLISignal(model_name=nli_model)
+        self._s3 = CosineSignal(model_name=cosine_model)
+
+    # ── Mode resolution ──────────────────────────────────────────────────────
+    @property
+    def active_mode(self) -> str:
+        """Resolves "auto" to "full" or "lite" based on Ollama availability."""
+        if self.mode == "auto":
+            return "full" if self._s1.is_available else "lite"
+        return self.mode
+
+    # ── Internal risk computation ─────────────────────────────────────────────
+    def _compute_risk(self, s1: float, s2: float, s3: float) -> float:
+        s3_risk = s3 if self.direction == "cosine_as_risk" else (1.0 - s3)
+        a = self.weights.get("alpha", 0.0)
+        b = self.weights.get("beta",  0.15)
+        g = self.weights.get("gamma", 0.85)
+        return float(np.clip(a * (1.0 - s1) + b * (1.0 - s2) + g * s3_risk, 0.0, 1.0))
+
+    # ── Public API ────────────────────────────────────────────────────────────
+    def score(self, question: str, answer: str, context: str) -> CGSResult:
+        """
+        Score a single RAG response.
+
+        Parameters
+        ----------
+        question : the user's question
+        answer   : the RAG system's generated answer
+        context  : the retrieved context passage(s)
+
+        Returns
+        -------
+        CGSResult with risk_score, is_hallucination, signals, and explain()
+        """
+        s2 = self._s2.score(question, answer, context)
+        s3 = self._s3.score(question, answer, context)
+        s1 = self._s1.score(question, answer, context) if self.active_mode == "full" else 0.5
+
+        risk    = self._compute_risk(s1, s2, s3)
+        signals = {
+            "s2_nli":     round(s2, 4),
+            "s3_cosine":  round(s3, 4),
+        }
+        if self.active_mode == "full":
+            signals["s1_logprob"] = round(s1, 4)
+
+        return CGSResult(
+            risk_score       = round(risk, 4),
+            is_hallucination = risk >= self.threshold,
+            threshold        = self.threshold,
+            mode             = self.active_mode,
+            signals          = signals,
+            direction        = self.direction,
+        )
+
+    # ── Batch scoring ─────────────────────────────────────────────────────────
+    def score_batch(
+        self,
+        questions: List[str],
+        answers:   List[str],
+        contexts:  List[str],
+    ) -> List[CGSResult]:
+        """
+        Score a batch of RAG responses.
+        Uses true batch inference for S2 and S3 (much faster than looping).
+
+        Returns
+        -------
+        List of CGSResult objects (same order as input).
+        """
+        n = len(questions)
+        assert len(answers) == n and len(contexts) == n, \
+            "questions, answers, contexts must have the same length"
+
+        # Batch signal computation
+        s2_arr = np.array(self._s2.score_batch(questions, answers, contexts))
+        s3_arr = np.array(self._s3.score_batch(questions, answers, contexts))
+
+        if self.active_mode == "full":
+            s1_arr = np.array(self._s1.score_batch(questions, answers, contexts))
+        else:
+            s1_arr = np.full(n, 0.5)
+
+        results = []
+        for i in range(n):
+            risk    = self._compute_risk(s1_arr[i], s2_arr[i], s3_arr[i])
+            signals = {
+                "s2_nli":    round(float(s2_arr[i]), 4),
+                "s3_cosine": round(float(s3_arr[i]), 4),
+            }
+            if self.active_mode == "full":
+                signals["s1_logprob"] = round(float(s1_arr[i]), 4)
+
+            results.append(CGSResult(
+                risk_score       = round(float(risk), 4),
+                is_hallucination = float(risk) >= self.threshold,
+                threshold        = self.threshold,
+                mode             = self.active_mode,
+                signals          = signals,
+                direction        = self.direction,
+            ))
+        return results
+
+    # ── Domain calibration ────────────────────────────────────────────────────
+    def calibrate(
+        self,
+        val_df:       pd.DataFrame,
+        question_col: str  = "question",
+        answer_col:   str  = "answer",
+        context_col:  str  = "context",
+        label_col:    str  = "label",
+        verbose:      bool = True,
+    ) -> dict:
+        """
+        Calibrate CGS weights, threshold, and direction to your domain.
+
+        Parameters
+        ----------
+        val_df       : labelled DataFrame.  Needs columns:
+                       question, answer, context, label (1=hallucinated, 0=grounded)
+        question_col : column name for questions
+        answer_col   : column name for answers
+        context_col  : column name for contexts
+        label_col    : column name for binary labels (1=hallucinated, 0=grounded)
+        verbose      : print progress and results
+
+        Returns
+        -------
+        dict with {"weights", "threshold", "direction", "auc", "n_samples"}
+
+        Side effects
+        ------------
+        Updates self.weights, self.threshold, self.direction, self._calibrated
+        """
+        if verbose:
+            print(f"[CGS] Calibrating on {len(val_df):,} samples …")
+
+        qs  = val_df[question_col].tolist()
+        ans = val_df[answer_col].tolist()
+        ctx = val_df[context_col].tolist()
+        lbl = val_df[label_col].values.astype(int)
+
+        if verbose: print("[CGS]   Computing S3 (cosine) …")
+        s3 = np.array(self._s3.score_batch(qs, ans, ctx))
+
+        if verbose: print("[CGS]   Computing S2 (NLI) …")
+        s2 = np.array(self._s2.score_batch(qs, ans, ctx))
+
+        s1 = None
+        if self.active_mode == "full":
+            if verbose: print("[CGS]   Computing S1 (token log-prob) …")
+            s1 = np.array(self._s1.score_batch(qs, ans, ctx))
+
+        # Direction auto-detection
+        self.direction = detect_signal_direction(s3, lbl)
+        if verbose:
+            print(f"[CGS]   Direction detected: {self.direction}")
+            mean_hal = s3[lbl == 1].mean()
+            mean_gnd = s3[lbl == 0].mean()
+            print(f"         E[s3|hal]={mean_hal:.3f}   E[s3|gnd]={mean_gnd:.3f}")
+
+        # Weight + threshold optimisation
+        self.weights, self.threshold, best_auc = optimise_weights(
+            s2, s3, lbl, self.direction, s1
+        )
+        self._calibrated = True
+
+        result = {
+            "weights":   self.weights,
+            "threshold": round(self.threshold, 3),
+            "direction": self.direction,
+            "auc":       best_auc,
+            "n_samples": len(val_df),
+        }
+
+        if verbose:
+            print(f"[CGS] Calibration complete:")
+            print(f"      Weights  : α={self.weights['alpha']:.2f}  "
+                  f"β={self.weights['beta']:.2f}  γ={self.weights['gamma']:.2f}")
+            print(f"      Threshold: τ = {self.threshold:.2f}")
+            print(f"      AUC      : {best_auc:.4f}")
+
+        return result
+
+    # ── Serialisation ─────────────────────────────────────────────────────────
+    def get_config(self) -> dict:
+        """Export calibrated configuration for storage / reproducibility."""
+        return {
+            "mode":        self.active_mode,
+            "weights":     self.weights,
+            "threshold":   self.threshold,
+            "direction":   self.direction,
+            "calibrated":  self._calibrated,
+        }
+
+    @classmethod
+    def from_config(cls, config: dict, **kwargs) -> "CGSDetector":
+        """Reconstruct a calibrated detector from a saved config dict."""
+        detector = cls(
+            mode      = config.get("mode", "auto"),
+            weights   = config.get("weights"),
+            threshold = config.get("threshold", _DEFAULT_THRESHOLD),
+            direction = config.get("direction", _DEFAULT_DIRECTION),
+            **kwargs,
+        )
+        detector._calibrated = config.get("calibrated", False)
+        return detector
+
+    # ── Repr ──────────────────────────────────────────────────────────────────
+    def __repr__(self) -> str:
+        return (
+            f"CGSDetector("
+            f"mode={self.active_mode!r}, "
+            f"τ={self.threshold:.2f}, "
+            f"direction={self.direction!r}, "
+            f"calibrated={self._calibrated})"
+        )

cgs_rag-0.1.0/cgs_rag/result.py

@@ -0,0 +1,100 @@
+"""
+CGSResult — the return type for every detector.score() call.
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict
+
+
+@dataclass
+class CGSResult:
+    """
+    Result of a single CGS hallucination detection call.
+
+    Attributes
+    ----------
+    risk_score       : float in [0, 1].  Higher = more likely hallucinated.
+    is_hallucination : True if risk_score >= threshold.
+    threshold        : decision threshold used for this call.
+    mode             : "lite" (S2+S3) or "full" (S1+S2+S3).
+    signals          : per-signal scores {"s2_nli": ..., "s3_cosine": ...}.
+    direction        : "cosine_as_risk" (HaluEval/adversarial) or
+                       "cosine_as_grounding" (natural RAG / TruthfulQA).
+    """
+
+    risk_score:       float
+    is_hallucination: bool
+    threshold:        float
+    mode:             str
+    signals:          Dict[str, float]
+    direction:        str
+
+    # ------------------------------------------------------------------ #
+    def explain(self) -> str:
+        """Return a human-readable breakdown of the score."""
+        verdict = "HALLUCINATION ⚠️" if self.is_hallucination else "GROUNDED ✅"
+        lines = [
+            f"CGS Risk Score : {self.risk_score:.3f}  →  {verdict}",
+            f"Threshold      : {self.threshold:.2f}  |  Mode: {self.mode}  |  Direction: {self.direction}",
+            "",
+            "Signal breakdown:",
+        ]
+
+        if "s1_logprob" in self.signals:
+            s1 = self.signals["s1_logprob"]
+            lines.append(f"  S1  Token confidence   : {s1:.3f}"
+                         f"  ({'high confidence' if s1 > 0.7 else 'low confidence'})")
+
+        s2 = self.signals.get("s2_nli", None)
+        if s2 is not None:
+            tag = "entailed" if s2 > 0.6 else ("neutral" if s2 > 0.35 else "contradicted")
+            lines.append(f"  S2  NLI faithfulness   : {s2:.3f}  ({tag})")
+
+        s3 = self.signals.get("s3_cosine", None)
+        if s3 is not None:
+            if self.direction == "cosine_as_risk":
+                tag = "suspicious — answer borrows context vocabulary" if s3 > 0.6 else "normal"
+            else:
+                tag = "well-grounded" if s3 > 0.6 else "semantically distant from context"
+            lines.append(f"  S3  Cosine similarity  : {s3:.3f}  ({tag})")
+
+        lines.append("")
+        lines.append("Interpretation:")
+
+        if self.is_hallucination:
+            reasons = []
+            if s3 is not None:
+                if self.direction == "cosine_as_risk" and s3 > 0.65:
+                    reasons.append("answer vocabulary closely mirrors context (adversarial distractor pattern)")
+                elif self.direction == "cosine_as_grounding" and s3 < 0.35:
+                    reasons.append("answer is semantically distant from the retrieved context")
+            if s2 is not None and s2 < 0.35:
+                reasons.append("NLI model does not support the answer given the context")
+            if reasons:
+                for r in reasons:
+                    lines.append(f"  → {r}")
+            else:
+                lines.append("  → Combined signal weight exceeds threshold.")
+        else:
+            lines.append("  → No strong hallucination signal detected.")
+
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------ #
+    def to_dict(self) -> dict:
+        """Serialisable dict for logging / storage."""
+        return {
+            "risk_score":       self.risk_score,
+            "is_hallucination": self.is_hallucination,
+            "threshold":        self.threshold,
+            "mode":             self.mode,
+            "direction":        self.direction,
+            **self.signals,
+        }
+
+    def __repr__(self) -> str:
+        return (
+            f"CGSResult(risk={self.risk_score:.3f}, "
+            f"hal={self.is_hallucination}, "
+            f"mode={self.mode!r})"
+        )

cgs_rag-0.1.0/cgs_rag/signals/__init__.py

@@ -0,0 +1,5 @@
+from .signal2_nli import NLISignal
+from .signal3_cosine import CosineSignal
+from .signal1_token import TokenLogProbSignal
+
+__all__ = ["NLISignal", "CosineSignal", "TokenLogProbSignal"]

cgs_rag-0.1.0/cgs_rag/signals/base.py

@@ -0,0 +1,45 @@
+"""Abstract base class for all CGS signals."""
+
+from abc import ABC, abstractmethod
+from typing import List
+
+
+class BaseSignal(ABC):
+    """
+    Every signal must implement:
+      - is_available  (property) : True if the signal's dependencies are installed/reachable
+      - score()                  : single sample → float in [0, 1]
+      - score_batch()            : list of samples → list of floats
+    """
+
+    @property
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Return True if this signal can be computed (deps installed, service up)."""
+
+    @abstractmethod
+    def score(self, question: str, answer: str, context: str) -> float:
+        """
+        Compute a raw signal value for one QA sample.
+
+        Returns
+        -------
+        float in [0, 1].
+        The *direction* of the signal (whether high = hallucinated or high = grounded)
+        is handled by CGSDetector, not here.
+        """
+
+    def score_batch(
+        self,
+        questions: List[str],
+        answers:   List[str],
+        contexts:  List[str],
+    ) -> List[float]:
+        """
+        Default batch implementation: loop over samples.
+        Subclasses may override for true batching (e.g. SentenceTransformer).
+        """
+        return [
+            self.score(q, a, c)
+            for q, a, c in zip(questions, answers, contexts)
+        ]

cgs_rag-0.1.0/cgs_rag/signals/signal1_token.py

@@ -0,0 +1,122 @@
+"""
+Signal 1 — Token Log-Probability Confidence.
+
+Queries a locally-running Ollama instance to obtain token-level log-probabilities
+for the generated answer given the question and context.  Converts the geometric
+mean log-prob to a confidence score in [0, 1].
+
+High score = model is highly confident = less likely to be a hallucination.
+Low score  = model is uncertain        = hallucination risk signal.
+
+Requirements
+------------
+- Ollama installed and running  (https://ollama.com)
+- The model pulled: ``ollama pull llama3.2``
+
+Graceful fallback
+-----------------
+If Ollama is not reachable, ``is_available`` returns False and the detector
+automatically falls back to Lite mode (S2+S3 only).  ``score()`` returns 0.5
+(neutral) so existing weights stay valid if called directly.
+"""
+
+import json
+import math
+import urllib.request
+import urllib.error
+from typing import List
+
+from .base import BaseSignal
+
+
+class TokenLogProbSignal(BaseSignal):
+    """Token log-probability confidence via Ollama."""
+
+    def __init__(
+        self,
+        model_name:  str = "llama3.2",
+        ollama_url:  str = "http://localhost:11434",
+        timeout_sec: int = 30,
+    ):
+        self.model_name  = model_name
+        self.ollama_url  = ollama_url.rstrip("/")
+        self.timeout_sec = timeout_sec
+
+    # ------------------------------------------------------------------ #
+    @property
+    def is_available(self) -> bool:
+        """Ping the Ollama /api/tags endpoint to check if the service is up."""
+        try:
+            req = urllib.request.Request(
+                f"{self.ollama_url}/api/tags",
+                method="GET",
+            )
+            with urllib.request.urlopen(req, timeout=2):
+                return True
+        except Exception:
+            return False
+
+    # ------------------------------------------------------------------ #
+    def score(self, question: str, answer: str, context: str) -> float:
+        """
+        Returns a calibrated confidence score in [0, 1].
+
+        Prompt structure:
+            Context: <context>
+            Question: <question>
+            Answer: <answer>
+
+        We ask Ollama to *continue* the prompt (generation mode) and collect
+        the log-probs of the answer tokens, computing the geometric mean
+        probability as the confidence score.
+        """
+        prompt = (
+            f"Context: {context}\n\n"
+            f"Question: {question}\n\n"
+            f"Answer: {answer}"
+        )
+        payload = json.dumps({
+            "model":  self.model_name,
+            "prompt": prompt,
+            "stream": False,
+            "options": {
+                "logprobs": True,
+                "temperature": 0,   # deterministic
+                "num_predict": 1,   # we only need the continuation start
+            },
+        }).encode("utf-8")
+
+        try:
+            req = urllib.request.Request(
+                f"{self.ollama_url}/api/generate",
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            with urllib.request.urlopen(req, timeout=self.timeout_sec) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+
+            logprobs: list = data.get("logprobs") or []
+            if not logprobs:
+                return 0.5  # logprobs not returned by this model
+
+            # Geometric mean probability = exp(mean log-prob)
+            avg_lp   = sum(logprobs) / len(logprobs)
+            conf     = math.exp(avg_lp)   # in (0, 1]
+            return max(0.0, min(1.0, conf))
+
+        except (urllib.error.URLError, json.JSONDecodeError, KeyError):
+            return 0.5   # graceful fallback — neutral confidence
+
+    # ------------------------------------------------------------------ #
+    def score_batch(
+        self,
+        questions: List[str],
+        answers:   List[str],
+        contexts:  List[str],
+    ) -> List[float]:
+        """Sequential batch (Ollama does not expose a true batch endpoint)."""
+        return [
+            self.score(q, a, c)
+            for q, a, c in zip(questions, answers, contexts)
+        ]

cgs_rag-0.1.0/cgs_rag/signals/signal2_nli.py

@@ -0,0 +1,115 @@
+"""
+Signal 2 — NLI Faithfulness Score.
+
+Uses cross-encoder/nli-deberta-v3-small to compute the probability that the
+retrieved context *entails* the generated answer.
+
+Hypothesis template (from thesis):
+    "The answer to '{question}' is: {answer}."
+
+Returns the ENTAILMENT probability in [0, 1].
+High score = answer is entailed by context = grounded.
+Low score  = answer contradicts or is unrelated to context = suspicious.
+"""
+
+from typing import List
+
+from .base import BaseSignal
+
+# DeBERTa NLI label sets differ by model variant — handle all possibilities.
+_ENTAIL_LABELS = {"entailment", "entail", "label_2", "yes"}
+_CONTRA_LABELS = {"contradiction", "contradict", "label_0", "no"}
+
+
+class NLISignal(BaseSignal):
+    """NLI entailment probability between context (premise) and answer (hypothesis)."""
+
+    def __init__(self, model_name: str = "cross-encoder/nli-deberta-v3-small"):
+        self.model_name = model_name
+        self._pipe = None  # lazy load
+
+    # ------------------------------------------------------------------ #
+    @property
+    def is_available(self) -> bool:
+        try:
+            import transformers  # noqa: F401
+            return True
+        except ImportError:
+            return False
+
+    # ------------------------------------------------------------------ #
+    def _load(self) -> None:
+        if self._pipe is None:
+            from transformers import pipeline
+            self._pipe = pipeline(
+                "text-classification",
+                model=self.model_name,
+                top_k=None,          # return all label scores
+                truncation=True,
+                max_length=512,
+            )
+
+    # ------------------------------------------------------------------ #
+    def score(self, question: str, answer: str, context: str) -> float:
+        """
+        Returns entailment probability in [0, 1].
+        Higher = answer is more faithfully supported by context.
+        """
+        self._load()
+        premise    = str(context)
+        hypothesis = f"The answer to '{question}' is: {answer}."
+
+        raw = self._pipe(
+            {"text": premise, "text_pair": hypothesis}
+        )
+        # raw is a list of dicts: [{"label": "ENTAILMENT", "score": 0.92}, ...]
+        return self._extract_entailment(raw)
+
+    # ------------------------------------------------------------------ #
+    def _extract_entailment(self, raw) -> float:
+        """
+        Robustly extract entailment probability from the pipeline output,
+        regardless of how the model labels its classes.
+        """
+        if not raw:
+            return 0.5
+
+        # Normalise to a list if wrapped in an extra list
+        items = raw[0] if (isinstance(raw[0], list)) else raw
+
+        label_score = {}
+        for item in items:
+            label_score[item["label"].lower()] = float(item["score"])
+
+        # Try to find entailment label directly
+        for lbl in _ENTAIL_LABELS:
+            if lbl in label_score:
+                return label_score[lbl]
+
+        # Fallback: 1 - contradiction_score (works when model uses 3 classes)
+        for lbl in _CONTRA_LABELS:
+            if lbl in label_score:
+                neutral_score = sum(
+                    v for k, v in label_score.items()
+                    if k not in _CONTRA_LABELS
+                )
+                return max(0.0, min(1.0, neutral_score))
+
+        # Last resort: assume highest-scoring label is entailment
+        return max(label_score.values()) if label_score else 0.5
+
+    # ------------------------------------------------------------------ #
+    def score_batch(
+        self,
+        questions: List[str],
+        answers:   List[str],
+        contexts:  List[str],
+    ) -> List[float]:
+        """Batch using transformers pipeline (handles internal batching)."""
+        self._load()
+        inputs = [
+            {"text": str(c), "text_pair": f"The answer to '{q}' is: {a}."}
+            for q, a, c in zip(questions, answers, contexts)
+        ]
+        results = self._pipe(inputs)
+        return [self._extract_entailment(r) for r in results]

cgs_rag-0.1.0/cgs_rag/signals/signal3_cosine.py

@@ -0,0 +1,85 @@
+"""
+Signal 3 — Semantic Attribution via Cosine Similarity.
+
+Uses all-MiniLM-L6-v2 (384-dim) to embed the answer and the retrieved context,
+then computes their cosine similarity.
+
+Direction note
+--------------
+This signal returns raw cosine similarity in [0, 1].
+The CGSDetector decides whether high cosine = risk or high cosine = grounding
+based on the calibrated ``direction`` attribute.
+"""
+
+from typing import List
+import numpy as np
+
+from .base import BaseSignal
+
+
+class CosineSignal(BaseSignal):
+    """Cosine similarity between answer embedding and context embedding."""
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+        self.model_name = model_name
+        self._model = None  # lazy load
+
+    # ------------------------------------------------------------------ #
+    @property
+    def is_available(self) -> bool:
+        try:
+            import sentence_transformers  # noqa: F401
+            return True
+        except ImportError:
+            return False
+
+    # ------------------------------------------------------------------ #
+    def _load(self) -> None:
+        if self._model is None:
+            from sentence_transformers import SentenceTransformer
+            self._model = SentenceTransformer(self.model_name)
+
+    # ------------------------------------------------------------------ #
+    def score(self, question: str, answer: str, context: str) -> float:
+        """
+        Returns cosine similarity between answer and context in [0, 1].
+        question is accepted but not used (kept for API consistency).
+        """
+        self._load()
+        embs = self._model.encode(
+            [str(answer), str(context)],
+            convert_to_numpy=True,
+            show_progress_bar=False,
+        )
+        a, b = embs[0], embs[1]
+        norm = np.linalg.norm(a) * np.linalg.norm(b)
+        if norm < 1e-10:
+            return 0.0
+        cosine = float(np.dot(a, b) / norm)
+        return max(0.0, min(1.0, cosine))
+
+    # ------------------------------------------------------------------ #
+    def score_batch(
+        self,
+        questions: List[str],
+        answers:   List[str],
+        contexts:  List[str],
+    ) -> List[float]:
+        """
+        True batch: encode all answers and contexts in two forward passes,
+        then compute pairwise cosine. Much faster than looping.
+        """
+        self._load()
+        all_answers  = [str(a) for a in answers]
+        all_contexts = [str(c) for c in contexts]
+
+        emb_ans = self._model.encode(all_answers,  convert_to_numpy=True, show_progress_bar=False)
+        emb_ctx = self._model.encode(all_contexts, convert_to_numpy=True, show_progress_bar=False)
+
+        # Row-wise cosine
+        norms_ans = np.linalg.norm(emb_ans, axis=1, keepdims=True).clip(min=1e-10)
+        norms_ctx = np.linalg.norm(emb_ctx, axis=1, keepdims=True).clip(min=1e-10)
+        emb_ans_n = emb_ans / norms_ans
+        emb_ctx_n = emb_ctx / norms_ctx
+        cosines = (emb_ans_n * emb_ctx_n).sum(axis=1)
+        return [max(0.0, min(1.0, float(c))) for c in cosines]

cgs_rag-0.1.0/cgs_rag.egg-info/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: cgs-rag
+Version: 0.1.0
+Summary: Composite Grounding Score: multi-signal hallucination detection for production RAG systems
+Author-email: Nishant Kumar <nishant.k@marmeto.com>
+License: MIT
+Project-URL: Homepage, https://github.com/nishant-k-marmeto/cgs-rag
+Project-URL: Repository, https://github.com/nishant-k-marmeto/cgs-rag
+Keywords: rag,retrieval-augmented-generation,hallucination,detection,nlp,llm,grounding,faithfulness
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Requires-Dist: torch>=2.0.0
+Requires-Dist: transformers>=4.36.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pandas>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: jupyter; extra == "dev"
+Requires-Dist: matplotlib; extra == "dev"
+Requires-Dist: seaborn; extra == "dev"

cgs_rag-0.1.0/cgs_rag.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+pyproject.toml
+cgs_rag/__init__.py
+cgs_rag/calibration.py
+cgs_rag/detector.py
+cgs_rag/result.py
+cgs_rag.egg-info/PKG-INFO
+cgs_rag.egg-info/SOURCES.txt
+cgs_rag.egg-info/dependency_links.txt
+cgs_rag.egg-info/requires.txt
+cgs_rag.egg-info/top_level.txt
+cgs_rag/signals/__init__.py
+cgs_rag/signals/base.py
+cgs_rag/signals/signal1_token.py
+cgs_rag/signals/signal2_nli.py
+cgs_rag/signals/signal3_cosine.py

cgs_rag-0.1.0/cgs_rag.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

cgs_rag-0.1.0/cgs_rag.egg-info/requires.txt

@@ -0,0 +1,13 @@
+torch>=2.0.0
+transformers>=4.36.0
+sentence-transformers>=2.2.0
+scikit-learn>=1.3.0
+numpy>=1.24.0
+pandas>=2.0.0
+
+[dev]
+pytest>=7.0
+pytest-cov
+jupyter
+matplotlib
+seaborn

cgs_rag-0.1.0/cgs_rag.egg-info/top_level.txt

@@ -0,0 +1 @@
+cgs_rag

cgs_rag-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires      = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name        = "cgs-rag"
+version     = "0.1.0"
+description = "Composite Grounding Score: multi-signal hallucination detection for production RAG systems"
+requires-python = ">=3.9"
+license     = {text = "MIT"}
+authors     = [{name = "Nishant Kumar", email = "nishant.k@marmeto.com"}]
+
+keywords = [
+  "rag", "retrieval-augmented-generation",
+  "hallucination", "detection",
+  "nlp", "llm", "grounding", "faithfulness",
+]
+
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+]
+
+dependencies = [
+  "torch>=2.0.0",
+  "transformers>=4.36.0",
+  "sentence-transformers>=2.2.0",
+  "scikit-learn>=1.3.0",
+  "numpy>=1.24.0",
+  "pandas>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=7.0",
+  "pytest-cov",
+  "jupyter",
+  "matplotlib",
+  "seaborn",
+]
+
+[project.urls]
+Homepage   = "https://github.com/nishant-k-marmeto/cgs-rag"
+Repository = "https://github.com/nishant-k-marmeto/cgs-rag"
+
+[tool.setuptools.packages.find]
+where   = ["."]
+include = ["cgs_rag*"]

cgs_rag-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+