ragsentry 0.1.0__py3-none-any.whl

ragsentry/__init__.py

@@ -0,0 +1 @@
+

ragsentry/__main__.py

@@ -0,0 +1,4 @@
+from ragsentry.cli import main
+
+if __name__ == "__main__":
+    main()

ragsentry/adapters/__init__.py

@@ -0,0 +1 @@
+

ragsentry/adapters/base.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+
+@dataclass
+class AdapterResponse:
+    """Standardized response structure returned by any RagSentry adapter."""
+
+    answer: str
+    contexts: list[str]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+    def to_dict(self) -> dict[str, Any]:
+        return{
+            "answer": self.answer,
+            "contexts": self.contexts,
+            "metadata": self.metadata,
+        }
+
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """Protocol defining the adapter contract for external RAG systems."""
+
+    def query(self, question: str) -> AdapterResponse:
+        """Query the target RAG system with a question and return an AdapterResponse."""
+        ...

ragsentry/adapters/http.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any
+import httpx
+
+
+from ragsentry.adapters.base import Adapter, AdapterResponse
+
+
+def extract_path(data: Any, path: str) -> Any:
+    """Extract a nested value using dotted syntax like 'data.answer' or 'results.0.text'."""
+    if not path:
+        return data
+
+    current = data
+    for part in path.split("."):
+        if isinstance(current, dict):
+            current = current.get(part)
+
+        elif isinstance(current, (list, tuple)) and part.isdigit():
+            idx = int(part)
+            current = current[idx] if 0<= idx < len(current) else None
+
+        else:
+            return None
+        
+        if current is None:
+            break
+    return current
+
+
+
+
+class HttpAdapter(Adapter):
+    """Adapter that queries an external RAG service over HTTP."""
+
+    def __init__(
+        self, 
+        url: str,
+        method: str = "POST",
+        headers: dict[str, str] | None = None,
+        question_key: str = "question",
+        answer_path: str = "answer",
+        contexts_path: str = "contexts",
+        metadata_path: str | None = "metadata",
+        timeout: float =30.0
+    ) -> None:
+        self.url = url
+        self.method = method.upper()
+        self.headers = headers or {"Content-Type": "application/json"}
+        self.question_key = question_key
+        self.answer_path = answer_path
+        self.contexts_path = contexts_path
+        self.metadata_path = metadata_path
+        self.timeout = timeout
+
+
+    def query(self, question: str) -> AdapterResponse:
+        payload = {self.question_key: question}
+
+        with httpx.Client(timeout=self.timeout) as client:
+            if self.method == "POST":
+                response = client.post(self.url ,json=payload, headers=self.headers)
+            elif self.method == "GET":
+                response = client.get(self.url ,params=payload, headers=self.headers)
+
+            else:
+                raise ValueError(f"Unsupported HTTP method: {self.method}")
+
+            
+            response.raise_for_status()
+            data = response.json()
+
+        
+        #extract the answer
+        raw_answer = extract_path(data, self.answer_path)
+        if raw_answer is None:
+            raise ValueError(
+                f"Failed to extract answer using path '{self.answer_path}' from response: {data}"
+            )
+
+        #extract the contexts
+        raw_contexts = extract_path(data, self.contexts_path)
+        if raw_contexts is None:
+            raw_contexts = []
+        elif not isinstance(raw_contexts, list):
+            raw_contexts = [raw_contexts]
+
+        contexts = [
+            c if isinstance(c, str) else str(c.get("text", c)) if isinstance(c, dict) else str(c)
+            for c in raw_contexts
+        ]
+
+        #extract metadata
+        metadata = {}
+        if self.metadata_path:
+            raw_meta = extract_path(data, self.metadata_path)
+            if isinstance(raw_meta, dict):
+                metadata = raw_meta
+
+        
+        return AdapterResponse(
+            answer=str(raw_answer),
+            contexts=contexts,
+            metadata=metadata
+        )

ragsentry/adapters/python_callable.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import importlib
+from typing import Any, Callable
+import os
+import sys
+
+from ragsentry.adapters.base import Adapter, AdapterResponse
+
+
+class PythonCallableAdapter(Adapter):
+    """Adapter that wraps an in-process Python callable."""
+
+
+    def __init__(self, target: Callable[[str], dict[str, Any] | AdapterResponse] | str) -> None:
+
+        if isinstance(target, str):
+            self._callable = self._load_callable(target)
+        elif callable(target):
+            self._callable = target
+
+        else:
+            raise TypeError(f"Target must be a callable or an import string ('module:func'), got {type(target)}")
+
+    
+
+    @staticmethod
+    def _load_callable(import_str: str) -> Callable[[str], Any]:
+        """Resolves an import string of format 'module.submodule:function_name'."""
+        if ":" not in import_str:
+            raise ValueError(
+                f"Invalid import string '{import_str}'. Expected format: 'package.module:function_name'"
+            )
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+            
+        module_path, func_name = import_str.split(":", 1)
+        module = importlib.import_module(module_path)
+        target = getattr(module, func_name)
+        if not callable(target):
+            raise TypeError(f"Resolved object '{import_str}' is not callable.")
+
+        return target
+
+
+    def query(self, question: str) -> AdapterResponse:
+
+        raw = self._callable(question)
+
+        if isinstance(raw, AdapterResponse):
+            return raw
+
+        if isinstance(raw, dict):
+            if "answer" not in raw or "contexts" not in raw:
+                raise ValueError(
+                    f"Callable returned a dict missing 'answer' or 'contexts': keys={list(raw.keys())}"
+                )
+            
+            contexts = raw["contexts"]
+            if not isinstance(contexts, list):
+                raise TypeError(f"'contexts' must be a list of strings, got {type(contexts)}")
+
+            
+            return AdapterResponse(
+                answer=str(raw["answer"]),
+                contexts=[str(c) for c in contexts],
+                metadata=raw.get("metadata", {}),
+            )
+
+
+        raise TypeError(
+            f"Expected callable to return dict or AdapterResponse, got {type(raw)}"
+        )

ragsentry/adapters/shell.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+import shlex
+import subprocess
+from typing import Any
+
+
+from ragsentry.adapters.base import Adapter, AdapterResponse
+
+
+class ShellAdapter(Adapter):
+    """Adapter that executes an external shell command or script."""
+
+    def __init__(
+        self,
+        command: str | list [str],
+        pass_as: str = "stdin", #stdin or arg
+        timeout: float = 30.0,
+        cwd: str | None = None,
+    ) -> None:
+
+        if isinstance(command, str):
+            self.command = shlex.split(command, posix=False)
+
+        else:
+            self.command = list(command)
+
+        
+        self.pass_as = pass_as.lower()
+        self.timeout = timeout
+        self.cwd = cwd
+
+
+        if self.pass_as not in ("stdin", "arg"):
+            raise ValueError(f"pass_as must be 'stdin' or 'arg', got '{pass_as}'")
+
+    
+    def query(self, question: str) -> AdapterResponse:
+        cmd = list(self.command)
+        input_data = None
+
+        if self.pass_as == "arg":
+            cmd.append(question)
+
+        else:
+            input_data = json.dumps({"question": question})
+
+        proc = subprocess.run(
+            cmd,
+            input=input_data,
+            capture_output=True,
+            text=True,
+            timeout=self.timeout,
+            cwd=self.cwd,
+        )
+        
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"Shell command failed with exit code {proc.returncode}.\nStderr: {proc.stderr}"
+            )
+        stdout_clean = proc.stdout.strip()
+        if not stdout_clean:
+            raise ValueError(f"Shell command produced no stdout output.\nStderr: {proc.stderr}")
+        try:
+            data: dict[str, Any] = json.loads(stdout_clean)
+        except json.JSONDecodeError as exc:
+            raise ValueError(
+                f"Shell adapter expected JSON output on stdout, got: {stdout_clean}"
+            ) from exc
+        if "answer" not in data or "contexts" not in data:
+            raise ValueError(
+                f"Shell command output missing 'answer' or 'contexts'. Keys found: {list(data.keys())}"
+            )
+        contexts = data["contexts"]
+        if not isinstance(contexts, list):
+            raise TypeError(f"'contexts' in shell output must be a list, got {type(contexts)}")
+        
+        
+        return AdapterResponse(
+            answer=str(data["answer"]),
+            contexts=[str(c) for c in contexts],
+            metadata=data.get("metadata", {}),
+        )

ragsentry/ci.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+from ragsentry.diff import RunDiff, compare_runs
+from ragsentry.storage.base import RunResult
+from ragsentry.storage import get_run_store
+
+
+
+@dataclass
+class CIGateResult:
+    """Outcome of a CI quality gate check."""
+    passed: bool
+    violations: list[str] = field(default_factory=list)
+    candidate_run: RunResult | None = None
+    baseline_run: RunResult | None = None
+    diff: RunDiff | None = None
+    thresholds: dict[str, float] = field(default_factory=dict)
+    max_regression: float | None = None
+    max_regressed_questions: int = 0
+
+
+    @property
+    def exit_code(self) -> int:
+        return 0 if self.passed else 1
+
+def parse_thresholds(threshold_inputs: list[str] | dict[str, float] | None) -> dict[str, float]:
+    """Parse threshold inputs from list of 'metric=val' strings or dict."""
+    if not threshold_inputs:
+        return {}
+
+    if isinstance(threshold_inputs, dict):
+        return {k: float(v) for k, v in threshold_inputs.items()}
+
+
+    result: dict[str, float] = {}
+    for item in threshold_inputs:
+        item = item.strip()
+        if not item:
+            continue
+        if "=" in item:
+            k, v = item.split("=", 1)
+            result[k.strip()] = float(v.strip())
+
+        elif ":" in item:
+            k, v = item.split(":", 1)
+            result[k.strip()] = float(v.strip())
+
+    
+    return result
+
+
+
+def evaluate_ci_gate(
+    candidate: RunResult | str | Path,
+    baseline: RunResult | str | Path | None = None,
+    thresholds: dict[str, float] | list[str] | None = None,
+    max_regression: float | None = None,
+    max_regressed_questions: int = 0,
+    tolerance: float = 0.001,
+    storage: str = "local",
+) -> CIGateResult:
+    """
+    Check if a candidate run satisfies CI quality criteria:
+    1. Absolute thresholds (e.g. faithfulness >= 0.8)
+    2. Regression checks against baseline (e.g. max drop <= 0.05, max regressed questions <= 0)
+    """
+
+    store = get_run_store(storage=storage)
+    violations: list[str] = []
+
+    if isinstance(candidate, (str, Path)):
+        cand_run = store.load(str(candidate))
+
+    else:
+        cand_run = candidate
+
+    
+    base_run: RunResult | None = None
+    diff: RunDiff | None = None
+
+    if baseline:
+        if isinstance(baseline, (str, Path)):
+            base_run = store.load(str(baseline))
+
+        else:
+            base_run = baseline
+        
+        diff = compare_runs(base_run, cand_run, tolerance=tolerance)
+
+
+    
+    parsed_thresh = parse_thresholds(thresholds)
+    avg_scores = cand_run.summary.get("average_scores", {})
+
+
+    # 1. Absolute Threshold Checks
+    for metric, min_val in parsed_thresh.items():
+        actual_val = avg_scores.get(metric)
+        if actual_val is None:
+            violations.append(
+                f"Missing metric '{metric}': expected >= {min_val:.4f}, but metric was not scored."
+            )
+        elif actual_val < min_val:
+            violations.append(
+                f"Threshold breach on '{metric}': score {actual_val:.4f} < required minimum {min_val:.4f}."
+            )
+
+    # 2. Baseline Regression Checks
+    if diff:
+        counts = diff.summary.get("counts", {})
+        regressed_q_count = counts.get("regressed", 0)
+
+
+        if regressed_q_count > max_regressed_questions:
+            violations.append(
+                f"Regression breach: {regressed_q_count} questions regressed (max allowed: {max_regressed_questions})."
+            )
+
+        
+
+        if max_regression is not None:
+            avg_deltas = diff.summary.get("average_deltas", {})
+            for m_name, d_info in avg_deltas.items():
+                delta = d_info.get("delta")
+                if delta is not None and delta < -max_regression:
+                    violations.append(
+                        f"Regression breach on '{m_name}': average score dropped by {abs(delta):.4f} "
+                        f"(max allowed drop: {max_regression:.4f})."
+                    )
+    passed = len(violations) == 0
+
+
+    return CIGateResult(
+        passed=passed,
+        violations=violations,
+        candidate_run=cand_run,
+        baseline_run=base_run,
+        diff=diff,
+        thresholds=parsed_thresh,
+        max_regression=max_regression,
+        max_regressed_questions=max_regressed_questions,
+    )
+
+