ragguard-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragguard_cli/__init__.py +6 -0
- ragguard_cli/cli.py +201 -0
- ragguard_cli/client.py +138 -0
- ragguard_cli/mcp_server.py +134 -0
- ragguard_cli-0.1.0.dist-info/METADATA +128 -0
- ragguard_cli-0.1.0.dist-info/RECORD +10 -0
- ragguard_cli-0.1.0.dist-info/WHEEL +5 -0
- ragguard_cli-0.1.0.dist-info/entry_points.txt +3 -0
- ragguard_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- ragguard_cli-0.1.0.dist-info/top_level.txt +1 -0
ragguard_cli/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""RAGGuard from the command line and from AI assistants (MCP): check that your RAG index matches your source documents."""
|
|
2
|
+
|
|
3
|
+
from .client import DEFAULT_URL, RagGuard, RagGuardError
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__all__ = ["DEFAULT_URL", "RagGuard", "RagGuardError", "__version__"]
|
ragguard_cli/cli.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""`ragguard`: check that your RAG index matches your source documents, from a terminal, a CI job or an AI assistant.
|
|
2
|
+
|
|
3
|
+
ragguard check --fail-on high run an audit, wait for it, print the result; exit 1 if open issues are at or above "high"
|
|
4
|
+
ragguard status the latest results, without running an audit
|
|
5
|
+
ragguard issues [--severity S] open issues, most severe first
|
|
6
|
+
ragguard issue ISSUE_ID one issue with its evidence and the recommended fix
|
|
7
|
+
ragguard document DOCUMENT_ID everything RAGGuard knows about one document
|
|
8
|
+
ragguard report the full report as Markdown
|
|
9
|
+
ragguard mcp run the MCP server for AI assistants (also: uvx ragguard-cli mcp)
|
|
10
|
+
|
|
11
|
+
Set RAGGUARD_API_KEY (create a key on the Integrations page of RAGGuard). RAGGUARD_URL defaults to the hosted service.
|
|
12
|
+
Exit codes: 0 fine, 1 open issues at or above --fail-on, 2 an error (bad key, unreachable server, failed audit).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import json
|
|
19
|
+
import sys
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from . import __version__
|
|
23
|
+
from .client import SEVERITIES, RagGuard, RagGuardError, worst_open
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _count_line(by_severity: dict[str, int]) -> str:
|
|
27
|
+
parts = [f"{sev} {by_severity[sev]}" for sev in SEVERITIES if by_severity.get(sev)]
|
|
28
|
+
return ", ".join(parts) if parts else "none"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _issue_line(i: dict[str, Any]) -> str:
|
|
32
|
+
where = f"{i.get('tenant_id')}/{i.get('document_id')}" if i.get("tenant_id") not in (None, "", "default") else i.get("document_id")
|
|
33
|
+
return f" [{i.get('severity')}] {i.get('title')} ({where}) id={i.get('id')}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _print_status(ov: dict[str, Any]) -> None:
|
|
37
|
+
docs, issues = ov.get("documents", {}), ov.get("issues", {})
|
|
38
|
+
score = docs.get("health_score")
|
|
39
|
+
print(f" health score : {'-' if score is None else f'{score}%'}")
|
|
40
|
+
print(f" documents : {docs.get('total', 0):,}")
|
|
41
|
+
print(f" open issues : {issues.get('open', 0):,} ({_count_line(issues.get('by_severity', {}))})")
|
|
42
|
+
run = ov.get("latest_run")
|
|
43
|
+
if run:
|
|
44
|
+
print(f" latest audit : {run.get('status')} at {run.get('finished_at') or run.get('started_at')}")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cmd_check(rg: RagGuard, args: argparse.Namespace) -> int:
|
|
48
|
+
progress = None if args.json else (lambda msg: print(f"... {msg}", file=sys.stderr))
|
|
49
|
+
run = rg.run_audit(timeout=args.timeout, on_progress=progress)
|
|
50
|
+
if run["status"] == "failed":
|
|
51
|
+
print(f"The audit failed: {run.get('error') or 'no reason given'}", file=sys.stderr)
|
|
52
|
+
return 2
|
|
53
|
+
ov = rg.status()
|
|
54
|
+
open_issues = rg.issues(limit=args.top)["items"] if args.top else []
|
|
55
|
+
over = worst_open(ov.get("issues", {}).get("by_severity", {}), args.fail_on)
|
|
56
|
+
if args.json:
|
|
57
|
+
print(json.dumps({"run": run, "overview": {k: ov.get(k) for k in ("documents", "issues")}, "open_issues": open_issues,
|
|
58
|
+
"at_or_above_fail_on": over}, indent=2, default=str))
|
|
59
|
+
else:
|
|
60
|
+
stats = run.get("stats") or {}
|
|
61
|
+
dur = stats.get("duration_seconds")
|
|
62
|
+
print(f"RAGGuard audit {run['id'][:8]}: {run['status']}" + (f" in {dur:.1f} s" if isinstance(dur, (int, float)) else ""))
|
|
63
|
+
_print_status(ov)
|
|
64
|
+
if open_issues:
|
|
65
|
+
print("Open issues, most severe first:")
|
|
66
|
+
for i in open_issues:
|
|
67
|
+
print(_issue_line(i))
|
|
68
|
+
if run["status"] == "partial":
|
|
69
|
+
print("This audit stopped part-way; the numbers cover the documents it reached.")
|
|
70
|
+
print(f"Open issues at or above {args.fail_on!r}: {over}")
|
|
71
|
+
return 1 if over else 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def cmd_status(rg: RagGuard, args: argparse.Namespace) -> int:
|
|
75
|
+
ov = rg.status()
|
|
76
|
+
if args.json:
|
|
77
|
+
print(json.dumps({k: ov.get(k) for k in ("documents", "issues", "latest_run")}, indent=2, default=str))
|
|
78
|
+
else:
|
|
79
|
+
print("RAGGuard status")
|
|
80
|
+
_print_status(ov)
|
|
81
|
+
return 0
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def cmd_issues(rg: RagGuard, args: argparse.Namespace) -> int:
|
|
85
|
+
out = rg.issues(severity=args.severity, limit=args.limit)
|
|
86
|
+
if args.json:
|
|
87
|
+
print(json.dumps(out, indent=2, default=str))
|
|
88
|
+
else:
|
|
89
|
+
print(f"{out['total']} open issue(s)" + (f" of severity {args.severity}" if args.severity else ""))
|
|
90
|
+
for i in out["items"]:
|
|
91
|
+
print(_issue_line(i))
|
|
92
|
+
return 0
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def cmd_issue(rg: RagGuard, args: argparse.Namespace) -> int:
|
|
96
|
+
i = rg.issue(args.issue_id)
|
|
97
|
+
if args.json:
|
|
98
|
+
print(json.dumps(i, indent=2, default=str))
|
|
99
|
+
return 0
|
|
100
|
+
print(f"[{i.get('severity')}] {i.get('title')} ({i.get('status')})")
|
|
101
|
+
print(f" document : {i.get('tenant_id')}/{i.get('document_id')}")
|
|
102
|
+
print(f" what : {i.get('detail')}")
|
|
103
|
+
action = i.get("action")
|
|
104
|
+
if isinstance(action, dict):
|
|
105
|
+
action = action.get("summary") or action.get("kind")
|
|
106
|
+
if action:
|
|
107
|
+
print(f" fix : {str(action).replace('_', ' ')}")
|
|
108
|
+
print(" evidence :")
|
|
109
|
+
print(" " + json.dumps(i.get("evidence"), indent=2, default=str).replace("\n", "\n "))
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def cmd_document(rg: RagGuard, args: argparse.Namespace) -> int:
|
|
114
|
+
d = rg.document(args.document_id, tenant=args.tenant)
|
|
115
|
+
if args.json:
|
|
116
|
+
print(json.dumps(d, indent=2, default=str))
|
|
117
|
+
return 0
|
|
118
|
+
doc = d.get("document") or {}
|
|
119
|
+
print(f"{doc.get('tenant_id')}/{doc.get('document_id')}: {doc.get('status')}")
|
|
120
|
+
for key in ("source_uri", "source_hash", "expected_chunks", "indexed_chunks"):
|
|
121
|
+
if doc.get(key) is not None:
|
|
122
|
+
print(f" {key:<15}: {doc[key]}")
|
|
123
|
+
issues = d.get("issues") or []
|
|
124
|
+
print(f" issues : {len(issues)}")
|
|
125
|
+
for i in issues:
|
|
126
|
+
print(_issue_line(i))
|
|
127
|
+
return 0
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def cmd_report(rg: RagGuard, args: argparse.Namespace) -> int:
|
|
131
|
+
sys.stdout.write(rg.report(args.format))
|
|
132
|
+
return 0
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def cmd_mcp(_rg: RagGuard | None, _args: argparse.Namespace) -> int:
|
|
136
|
+
try:
|
|
137
|
+
from .mcp_server import serve
|
|
138
|
+
except ImportError:
|
|
139
|
+
print('The MCP server needs the "mcp" package: pip install --upgrade ragguard-cli', file=sys.stderr)
|
|
140
|
+
return 2
|
|
141
|
+
serve()
|
|
142
|
+
return 0
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
146
|
+
p = argparse.ArgumentParser(prog="ragguard", description="Check that your RAG index matches your source documents (RAGGuard).")
|
|
147
|
+
p.add_argument("--version", action="version", version=f"ragguard-cli {__version__}")
|
|
148
|
+
p.add_argument("--url", help="the RAGGuard server (default: $RAGGUARD_URL, else the hosted service)")
|
|
149
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
150
|
+
|
|
151
|
+
c = sub.add_parser("check", help="run an audit and wait for the result")
|
|
152
|
+
c.add_argument("--fail-on", choices=SEVERITIES, default="high", help="exit 1 when open issues are at or above this (default: high)")
|
|
153
|
+
c.add_argument("--top", type=int, default=10, help="how many open issues to list (default 10, 0 for none)")
|
|
154
|
+
c.add_argument("--timeout", type=float, default=1800.0, help="seconds to wait for the audit (default 1800)")
|
|
155
|
+
c.add_argument("--json", action="store_true", help="print JSON instead of text")
|
|
156
|
+
c.set_defaults(fn=cmd_check)
|
|
157
|
+
|
|
158
|
+
s = sub.add_parser("status", help="the latest results, without running an audit")
|
|
159
|
+
s.add_argument("--json", action="store_true")
|
|
160
|
+
s.set_defaults(fn=cmd_status)
|
|
161
|
+
|
|
162
|
+
i = sub.add_parser("issues", help="open issues, most severe first")
|
|
163
|
+
i.add_argument("--severity", choices=SEVERITIES)
|
|
164
|
+
i.add_argument("--limit", type=int, default=50)
|
|
165
|
+
i.add_argument("--json", action="store_true")
|
|
166
|
+
i.set_defaults(fn=cmd_issues)
|
|
167
|
+
|
|
168
|
+
one = sub.add_parser("issue", help="one issue with its evidence and the recommended fix")
|
|
169
|
+
one.add_argument("issue_id")
|
|
170
|
+
one.add_argument("--json", action="store_true")
|
|
171
|
+
one.set_defaults(fn=cmd_issue)
|
|
172
|
+
|
|
173
|
+
d = sub.add_parser("document", help="everything RAGGuard knows about one document")
|
|
174
|
+
d.add_argument("document_id")
|
|
175
|
+
d.add_argument("--tenant", help="the tenant, when the same ID exists for several")
|
|
176
|
+
d.add_argument("--json", action="store_true")
|
|
177
|
+
d.set_defaults(fn=cmd_document)
|
|
178
|
+
|
|
179
|
+
r = sub.add_parser("report", help="the full report")
|
|
180
|
+
r.add_argument("--format", choices=["md", "json", "csv"], default="md")
|
|
181
|
+
r.set_defaults(fn=cmd_report)
|
|
182
|
+
|
|
183
|
+
m = sub.add_parser("mcp", help="run the MCP server over stdio, for AI assistants")
|
|
184
|
+
m.set_defaults(fn=cmd_mcp)
|
|
185
|
+
return p
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def main(argv: list[str] | None = None, *, client: RagGuard | None = None) -> int:
|
|
189
|
+
args = build_parser().parse_args(argv)
|
|
190
|
+
if args.cmd == "mcp":
|
|
191
|
+
return cmd_mcp(None, args)
|
|
192
|
+
try:
|
|
193
|
+
rg = client or RagGuard(url=args.url)
|
|
194
|
+
return args.fn(rg, args)
|
|
195
|
+
except RagGuardError as exc:
|
|
196
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
197
|
+
return 2
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
if __name__ == "__main__":
|
|
201
|
+
sys.exit(main())
|
ragguard_cli/client.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""A small client for the RAGGuard HTTP API, used by the `ragguard` command and the MCP server.
|
|
2
|
+
|
|
3
|
+
It authenticates with an API key (create one on the Integrations page of RAGGuard), which can run audits and read results but
|
|
4
|
+
cannot read connection settings or create other keys. Every call is a plain HTTPS request; nothing is cached on disk.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
DEFAULT_URL = "https://rag-integrity-platform.vercel.app"
|
|
17
|
+
SEVERITIES = ("critical", "high", "medium", "low", "info") # most severe first
|
|
18
|
+
FINAL = ("completed", "failed")
|
|
19
|
+
MAX_STRETCHES = 500 # a safety stop for a queued audit: a 4-million-vector index is a few dozen stretches
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class RagGuardError(Exception):
|
|
23
|
+
"""Anything that went wrong talking to RAGGuard, in words a person can act on."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class RagGuard:
|
|
27
|
+
def __init__(self, url: str | None = None, api_key: str | None = None, *, http: httpx.Client | None = None,
|
|
28
|
+
timeout: float = 330.0) -> None:
|
|
29
|
+
self.url = (url or os.environ.get("RAGGUARD_URL") or DEFAULT_URL).rstrip("/")
|
|
30
|
+
self.api_key = api_key if api_key is not None else os.environ.get("RAGGUARD_API_KEY", "")
|
|
31
|
+
if not self.api_key and http is None:
|
|
32
|
+
raise RagGuardError("No API key. Create one on the Integrations page of RAGGuard and set RAGGUARD_API_KEY.")
|
|
33
|
+
headers = {"Authorization": f"Bearer {self.api_key}", "User-Agent": "ragguard-cli"}
|
|
34
|
+
if http is not None: # tests pass an in-process client
|
|
35
|
+
http.headers.update(headers)
|
|
36
|
+
self.http = http
|
|
37
|
+
else:
|
|
38
|
+
# an audit started on a serverless host can run inside the request for several minutes
|
|
39
|
+
self.http = httpx.Client(base_url=self.url, headers=headers, timeout=timeout)
|
|
40
|
+
|
|
41
|
+
# ------------------------------------------------------------------ plumbing
|
|
42
|
+
def _call(self, method: str, path: str, **kw: Any) -> httpx.Response:
|
|
43
|
+
try:
|
|
44
|
+
r = self.http.request(method, path, **kw)
|
|
45
|
+
except httpx.HTTPError as exc:
|
|
46
|
+
raise RagGuardError(f"Could not reach RAGGuard at {self.url}: {type(exc).__name__}: {exc}") from exc
|
|
47
|
+
if r.status_code == 401:
|
|
48
|
+
raise RagGuardError("RAGGuard refused the API key. Check RAGGUARD_API_KEY, or create a new key on the Integrations page.")
|
|
49
|
+
if r.status_code >= 400:
|
|
50
|
+
try:
|
|
51
|
+
detail = r.json().get("detail")
|
|
52
|
+
except ValueError:
|
|
53
|
+
detail = r.text[:300]
|
|
54
|
+
raise RagGuardError(f"RAGGuard answered {r.status_code}: {detail}")
|
|
55
|
+
return r
|
|
56
|
+
|
|
57
|
+
def _json(self, method: str, path: str, **kw: Any) -> Any:
|
|
58
|
+
return self._call(method, path, **kw).json()
|
|
59
|
+
|
|
60
|
+
# ------------------------------------------------------------------ reads
|
|
61
|
+
def status(self) -> dict[str, Any]:
|
|
62
|
+
"""The overview: health score, document and issue counts, and the latest audit."""
|
|
63
|
+
return self._json("GET", "/api/overview")
|
|
64
|
+
|
|
65
|
+
def audit(self, run_id: str) -> dict[str, Any]:
|
|
66
|
+
return self._json("GET", f"/api/audits/{run_id}")
|
|
67
|
+
|
|
68
|
+
def issues(self, *, severity: str | None = None, status: str = "open", limit: int = 50) -> dict[str, Any]:
|
|
69
|
+
params: dict[str, Any] = {"status": status, "limit": limit}
|
|
70
|
+
if severity:
|
|
71
|
+
params["severity"] = severity
|
|
72
|
+
out = self._json("GET", "/api/issues", params=params)
|
|
73
|
+
out["items"].sort(key=lambda i: SEVERITIES.index(i.get("severity", "info")) if i.get("severity") in SEVERITIES else 9)
|
|
74
|
+
return out
|
|
75
|
+
|
|
76
|
+
def issue(self, issue_id: str) -> dict[str, Any]:
|
|
77
|
+
return self._json("GET", f"/api/issues/{issue_id}")
|
|
78
|
+
|
|
79
|
+
def document(self, document_id: str, tenant: str | None = None) -> dict[str, Any]:
|
|
80
|
+
"""One document's state and issues. Without a tenant, the document is looked up by its ID first."""
|
|
81
|
+
if tenant is None:
|
|
82
|
+
found = self._json("GET", "/api/documents", params={"q": document_id, "status": "all", "limit": 50})["items"]
|
|
83
|
+
exact = [d for d in found if d.get("document_id") == document_id]
|
|
84
|
+
if not exact:
|
|
85
|
+
raise RagGuardError(f"RAGGuard has no document with the ID {document_id!r}. Run an audit first, or check the ID.")
|
|
86
|
+
if len({d.get("tenant_id") for d in exact}) > 1:
|
|
87
|
+
tenants = ", ".join(sorted(str(d.get("tenant_id")) for d in exact))
|
|
88
|
+
raise RagGuardError(f"{document_id!r} exists for several tenants ({tenants}). Say which one.")
|
|
89
|
+
tenant = exact[0]["tenant_id"]
|
|
90
|
+
return self._json("GET", "/api/document", params={"tenant": tenant, "document_id": document_id})
|
|
91
|
+
|
|
92
|
+
def report(self, fmt: str = "md") -> str:
|
|
93
|
+
return self._call("GET", "/api/report", params={"format": fmt}).text
|
|
94
|
+
|
|
95
|
+
# ------------------------------------------------------------------ the audit
|
|
96
|
+
def run_audit(self, *, timeout: float = 1800.0, poll: float = 3.0,
|
|
97
|
+
on_progress: Callable[[str], None] | None = None) -> dict[str, Any]:
|
|
98
|
+
"""Start an audit and wait for it to end; returns the finished run (with its stats).
|
|
99
|
+
|
|
100
|
+
On a server with a queue (RAGI_AUDIT_QUEUE), the audit waits for a worker; like the dashboard, this then does the worker's job
|
|
101
|
+
itself, one stretch at a time, until the audit is over. A real worker is never in the way: whoever claims a stretch first gets
|
|
102
|
+
it, and this call then simply waits.
|
|
103
|
+
"""
|
|
104
|
+
say = on_progress or (lambda _msg: None)
|
|
105
|
+
deadline = time.monotonic() + timeout
|
|
106
|
+
start = self._json("POST", "/api/audits", json={})
|
|
107
|
+
run_id, status = start["run_id"], start["status"]
|
|
108
|
+
say(f"audit {run_id[:8]} {status}")
|
|
109
|
+
if status == "queued":
|
|
110
|
+
for _ in range(MAX_STRETCHES):
|
|
111
|
+
if time.monotonic() > deadline:
|
|
112
|
+
raise RagGuardError(f"The audit is still going after {int(timeout)} s. Check it later with `ragguard status`.")
|
|
113
|
+
out = self._json("POST", "/api/worker/step")
|
|
114
|
+
if out.get("run_id"):
|
|
115
|
+
run_id = out["run_id"]
|
|
116
|
+
if out.get("percent") is not None:
|
|
117
|
+
say(f"{out['percent']:.0f}% done")
|
|
118
|
+
if out["status"] == "partial" and out.get("next"):
|
|
119
|
+
continue # this stretch stopped part-way and queued the next one
|
|
120
|
+
if out["status"] != "idle":
|
|
121
|
+
break
|
|
122
|
+
status = self.audit(run_id)["status"] # idle: a real worker has it, so wait for it
|
|
123
|
+
if status in FINAL or status == "partial":
|
|
124
|
+
break
|
|
125
|
+
time.sleep(poll)
|
|
126
|
+
while True:
|
|
127
|
+
run = self.audit(run_id)
|
|
128
|
+
if run["status"] in FINAL or run["status"] == "partial":
|
|
129
|
+
return run
|
|
130
|
+
if time.monotonic() > deadline:
|
|
131
|
+
raise RagGuardError(f"The audit is still going after {int(timeout)} s. Check it later with `ragguard status`.")
|
|
132
|
+
time.sleep(poll)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def worst_open(by_severity: dict[str, int], at_least: str) -> int:
|
|
136
|
+
"""How many open issues are at or above a severity."""
|
|
137
|
+
cut = SEVERITIES.index(at_least)
|
|
138
|
+
return sum(n for sev, n in (by_severity or {}).items() if sev in SEVERITIES and SEVERITIES.index(sev) <= cut)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""The RAGGuard MCP server: lets an AI assistant (Claude Code, Cursor, Claude Desktop, VS Code...) check a RAG index.
|
|
2
|
+
|
|
3
|
+
Run it with `ragguard mcp` (stdio). It needs RAGGUARD_API_KEY, and RAGGUARD_URL for a self-hosted server. Every tool calls the
|
|
4
|
+
RAGGuard API with that key, which can run audits and read results but cannot read connection settings or create other keys.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import functools
|
|
10
|
+
import logging
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
try: # mcp 2.x renamed FastMCP to MCPServer; the parts used here (name, instructions, .tool(), .run()) are the same in both
|
|
14
|
+
from mcp.server.mcpserver import MCPServer as _Server
|
|
15
|
+
except ImportError: # mcp 1.x
|
|
16
|
+
from mcp.server.fastmcp import FastMCP as _Server
|
|
17
|
+
|
|
18
|
+
from .client import SEVERITIES, RagGuard, RagGuardError
|
|
19
|
+
|
|
20
|
+
INSTRUCTIONS = (
|
|
21
|
+
"RAGGuard checks that a RAG (retrieval-augmented generation) index matches its source documents: missing, stale, "
|
|
22
|
+
"half-indexed, duplicate, empty, deleted-but-still-searchable and wrong-tenant chunks. Use it after changing ingestion, "
|
|
23
|
+
"chunking or embedding code, after a re-index, or when a user says the RAG app gave a wrong or outdated answer. "
|
|
24
|
+
"Start with ragguard_status; run ragguard_run_audit for fresh results; read ragguard_get_issue for the evidence and the fix."
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
mcp = _Server("RAGGuard", instructions=INSTRUCTIONS)
|
|
28
|
+
_client: RagGuard | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def client() -> RagGuard:
|
|
32
|
+
global _client
|
|
33
|
+
if _client is None:
|
|
34
|
+
_client = RagGuard()
|
|
35
|
+
return _client
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _issue(i: dict[str, Any]) -> dict[str, Any]:
|
|
39
|
+
return {k: i.get(k) for k in ("id", "severity", "type", "title", "tenant_id", "document_id", "status", "detail")}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _summary(ov: dict[str, Any]) -> dict[str, Any]:
|
|
43
|
+
run = ov.get("latest_run") or {}
|
|
44
|
+
return {
|
|
45
|
+
"health_score": (ov.get("documents") or {}).get("health_score"),
|
|
46
|
+
"documents": (ov.get("documents") or {}).get("total"),
|
|
47
|
+
"open_issues": (ov.get("issues") or {}).get("open"),
|
|
48
|
+
"open_issues_by_severity": (ov.get("issues") or {}).get("by_severity"),
|
|
49
|
+
"latest_audit": {k: run.get(k) for k in ("id", "status", "started_at", "finished_at")} if run else None,
|
|
50
|
+
"audit_running": ov.get("audit_running"),
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _guard(fn):
|
|
55
|
+
"""Turn an API error into a message the assistant can act on, instead of a stack trace. functools.wraps keeps the
|
|
56
|
+
signature, which the MCP framework reads to describe the tool's parameters."""
|
|
57
|
+
returns_text = fn.__annotations__.get("return") in (str, "str")
|
|
58
|
+
|
|
59
|
+
@functools.wraps(fn)
|
|
60
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
61
|
+
try:
|
|
62
|
+
return fn(*args, **kwargs)
|
|
63
|
+
except RagGuardError as exc:
|
|
64
|
+
return f"error: {exc}" if returns_text else {"error": str(exc)}
|
|
65
|
+
|
|
66
|
+
return wrapper
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@mcp.tool()
|
|
70
|
+
@_guard
|
|
71
|
+
def ragguard_status() -> dict[str, Any]:
|
|
72
|
+
"""The current health of the RAG index from the latest audit: health score (share of documents whose index matches the
|
|
73
|
+
latest source exactly), document count, open issues by severity, and when the latest audit ran. Fast; runs no audit."""
|
|
74
|
+
return _summary(client().status())
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@mcp.tool()
|
|
78
|
+
@_guard
|
|
79
|
+
def ragguard_run_audit() -> dict[str, Any]:
|
|
80
|
+
"""Run a fresh audit of the RAG index against its source documents and wait for it to finish (can take minutes on a large
|
|
81
|
+
index). Returns the audit's result and the open issues, most severe first. Use after ingestion or embedding changes."""
|
|
82
|
+
rg = client()
|
|
83
|
+
run = rg.run_audit()
|
|
84
|
+
stats = run.get("stats") or {}
|
|
85
|
+
out = {
|
|
86
|
+
"audit": {"id": run.get("id"), "status": run.get("status"), "error": run.get("error"),
|
|
87
|
+
"documents_checked": stats.get("documents_total"), "findings": stats.get("findings_total"),
|
|
88
|
+
"health_score": stats.get("health_score"), "duration_seconds": stats.get("duration_seconds")},
|
|
89
|
+
**_summary(rg.status()),
|
|
90
|
+
"top_open_issues": [_issue(i) for i in rg.issues(limit=15)["items"]],
|
|
91
|
+
}
|
|
92
|
+
if run.get("status") == "partial":
|
|
93
|
+
out["note"] = "The audit stopped part-way: the numbers cover the documents it reached."
|
|
94
|
+
return out
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@mcp.tool()
|
|
98
|
+
@_guard
|
|
99
|
+
def ragguard_list_issues(severity: str | None = None, limit: int = 25) -> dict[str, Any]:
|
|
100
|
+
"""Open integrity issues, most severe first. Optional severity filter: critical, high, medium, low or info. Each issue names
|
|
101
|
+
its type (for example stale_embedding, orphan_vector, missing_embedding, tenant_mismatch), document and ID."""
|
|
102
|
+
if severity and severity not in SEVERITIES:
|
|
103
|
+
return {"error": f"severity must be one of {', '.join(SEVERITIES)}"}
|
|
104
|
+
out = client().issues(severity=severity, limit=max(1, min(limit, 200)))
|
|
105
|
+
return {"total": out["total"], "issues": [_issue(i) for i in out["items"]]}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@mcp.tool()
|
|
109
|
+
@_guard
|
|
110
|
+
def ragguard_get_issue(issue_id: str) -> dict[str, Any]:
|
|
111
|
+
"""One issue in full: what is wrong, the evidence (expected versus actual values RAGGuard read), the recommended repair
|
|
112
|
+
action and its history. Use the ID from ragguard_list_issues or ragguard_run_audit."""
|
|
113
|
+
return client().issue(issue_id)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@mcp.tool()
|
|
117
|
+
@_guard
|
|
118
|
+
def ragguard_check_document(document_id: str, tenant: str | None = None) -> dict[str, Any]:
|
|
119
|
+
"""Everything RAGGuard knows about one document: whether its source, the pipeline's records and the index agree, its
|
|
120
|
+
chunk counts and hashes, and its open issues. Use when a user says an answer about that document is wrong or outdated."""
|
|
121
|
+
return client().document(document_id, tenant=tenant)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@mcp.tool()
|
|
125
|
+
@_guard
|
|
126
|
+
def ragguard_report() -> str:
|
|
127
|
+
"""The full audit report as Markdown: totals, health by tenant, and every open issue with its evidence."""
|
|
128
|
+
return client().report("md")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def serve() -> None:
|
|
132
|
+
# stdout carries the MCP protocol; keep the request-by-request HTTP log out of the assistant's logs too
|
|
133
|
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
134
|
+
mcp.run() # stdio
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ragguard-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Check that your RAG index matches your source documents: a command line tool and an MCP server for AI assistants (RAGGuard).
|
|
5
|
+
Author: RAGGuard
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://rag-integrity-platform.vercel.app
|
|
8
|
+
Project-URL: Documentation, https://rag-integrity-platform.vercel.app/guides/use-ragguard-from-your-ai-assistant
|
|
9
|
+
Project-URL: Source, https://github.com/kishore600/rag-integrity-platform/tree/main/clients/python
|
|
10
|
+
Keywords: rag,retrieval-augmented-generation,vector-database,qdrant,pgvector,embeddings,mcp,data-integrity,llm
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Database
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: httpx>=0.27
|
|
21
|
+
Requires-Dist: mcp>=1.2
|
|
22
|
+
Provides-Extra: mcp
|
|
23
|
+
Requires-Dist: mcp>=1.2; extra == "mcp"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# ragguard-cli
|
|
27
|
+
|
|
28
|
+
Check that your RAG index matches your source documents, from a terminal, a CI job or an AI assistant.
|
|
29
|
+
|
|
30
|
+
[RAGGuard](https://rag-integrity-platform.vercel.app) compares the files in Amazon S3 with your Qdrant or pgvector index, and
|
|
31
|
+
optionally your ingestion pipeline's records, and finds what silently drifted: documents that were never embedded, stale embeddings
|
|
32
|
+
after a document changed, two versions of a document answering at once, deleted documents that are still searchable, half-indexed
|
|
33
|
+
documents, duplicate chunks, empty vectors and chunks tagged with the wrong tenant. It only reads your systems.
|
|
34
|
+
|
|
35
|
+
This package gives you:
|
|
36
|
+
|
|
37
|
+
- the `ragguard` command, for people and CI jobs;
|
|
38
|
+
- an MCP server (`ragguard mcp`), so Claude Code, Cursor, Claude Desktop, VS Code and other AI assistants can run the checks.
|
|
39
|
+
|
|
40
|
+
<!-- mcp-name: io.github.kishore600/ragguard -->
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install ragguard-cli
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Or run it without installing, with [uv](https://docs.astral.sh/uv/): `uvx ragguard-cli status`.
|
|
49
|
+
|
|
50
|
+
Create an API key in RAGGuard on the **Integrations** page and set it:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
export RAGGUARD_API_KEY=ragi_your_key_here
|
|
54
|
+
# export RAGGUARD_URL=https://your-own-ragguard.example.com # only for a self-hosted server
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The key can run audits and read results. It cannot read your connection settings or create other keys.
|
|
58
|
+
|
|
59
|
+
## Use it
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
ragguard check --fail-on high # run an audit and wait; exit code 1 if open issues are high or critical
|
|
63
|
+
ragguard status # the latest results, without running an audit
|
|
64
|
+
ragguard issues --severity high # open issues, most severe first
|
|
65
|
+
ragguard issue ISSUE_ID # one issue with its evidence and the recommended fix
|
|
66
|
+
ragguard document refund-policy.pdf
|
|
67
|
+
ragguard report > report.md
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Add `--json` to `check`, `status`, `issues`, `issue` and `document` for machine-readable output. Exit codes: `0` fine, `1` open issues at
|
|
71
|
+
or above `--fail-on`, `2` an error.
|
|
72
|
+
|
|
73
|
+
### In CI
|
|
74
|
+
|
|
75
|
+
```yaml
|
|
76
|
+
- run: pip install ragguard-cli
|
|
77
|
+
- run: ragguard check --fail-on high
|
|
78
|
+
env:
|
|
79
|
+
RAGGUARD_API_KEY: ${{ secrets.RAGGUARD_API_KEY }}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Connect your AI assistant (MCP)
|
|
83
|
+
|
|
84
|
+
**Claude Code:**
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
claude mcp add ragguard -e RAGGUARD_API_KEY=ragi_your_key_here -- ragguard mcp
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
**Cursor, Claude Desktop and other MCP clients:**
|
|
91
|
+
|
|
92
|
+
```json
|
|
93
|
+
{
|
|
94
|
+
"mcpServers": {
|
|
95
|
+
"ragguard": {
|
|
96
|
+
"command": "ragguard",
|
|
97
|
+
"args": ["mcp"],
|
|
98
|
+
"env": { "RAGGUARD_API_KEY": "ragi_your_key_here" }
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Without installing anything first, use `"command": "uvx"` and `"args": ["ragguard-cli", "mcp"]` instead.
|
|
105
|
+
|
|
106
|
+
Tools: `ragguard_status`, `ragguard_run_audit`, `ragguard_list_issues`, `ragguard_get_issue`, `ragguard_check_document`, `ragguard_report`.
|
|
107
|
+
|
|
108
|
+
### A rule for your assistant
|
|
109
|
+
|
|
110
|
+
Paste this into `AGENTS.md`, `CLAUDE.md`, `.cursorrules` or `.github/copilot-instructions.md`:
|
|
111
|
+
|
|
112
|
+
```markdown
|
|
113
|
+
## RAG index integrity (RAGGuard)
|
|
114
|
+
- After changing ingestion, chunking or embedding code, or after a bulk re-index, run
|
|
115
|
+
`ragguard check --fail-on high` (or the MCP tool `ragguard_run_audit`) and fix every open
|
|
116
|
+
issue it reports before saying the work is done.
|
|
117
|
+
- Do not call an ingestion change finished while RAGGuard reports missing, stale, duplicate,
|
|
118
|
+
half-indexed or wrong-tenant chunks.
|
|
119
|
+
- When a user reports a wrong or outdated answer, run `ragguard document <document-id>`
|
|
120
|
+
before changing prompts or models: the index is often the cause.
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Learn more
|
|
124
|
+
|
|
125
|
+
- [Guides](https://rag-integrity-platform.vercel.app/guides): eight ways a RAG index drifts, with a check for each
|
|
126
|
+
- [Use RAGGuard from your AI assistant](https://rag-integrity-platform.vercel.app/guides/use-ragguard-from-your-ai-assistant)
|
|
127
|
+
|
|
128
|
+
MIT licensed.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
ragguard_cli/__init__.py,sha256=6H7t7NdVq4qCnW1j3kocGvqrqtX4gVRIxfYcDvbCUb4,281
|
|
2
|
+
ragguard_cli/cli.py,sha256=chgLgNSMjgEOfbFBNyq5SC4bJsQt5sOG8QsLMHm-Nn4,8865
|
|
3
|
+
ragguard_cli/client.py,sha256=Eu1-qg4ogIQT3KeD_qLeqge7oSZEx_0fk41qF3VE3Rg,7440
|
|
4
|
+
ragguard_cli/mcp_server.py,sha256=Ev9luHFfLDzu1duL0x-97BfbZplYFf32eieipBWUnWQ,6044
|
|
5
|
+
ragguard_cli-0.1.0.dist-info/licenses/LICENSE,sha256=PBP75wIcNz6axJFgiOTOiQkJTKiC2b6Jyw9lZse3dvw,1088
|
|
6
|
+
ragguard_cli-0.1.0.dist-info/METADATA,sha256=h4bsuA6gtPJJt1IPjXt3EXYbRjLhQFiB2sRdzgLZy2c,4922
|
|
7
|
+
ragguard_cli-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
8
|
+
ragguard_cli-0.1.0.dist-info/entry_points.txt,sha256=Wa9hHJL_XwMkoIRGqGxQqZJH64otnjpKOS7Fs7PzO2I,88
|
|
9
|
+
ragguard_cli-0.1.0.dist-info/top_level.txt,sha256=kPYyO9ZgrZpXVuR_1JFyGhbWRibJSaMEqIg79Xnbwm8,13
|
|
10
|
+
ragguard_cli-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 kishore600
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ragguard_cli
|