ragradar 0.1.0__tar.gz

ragradar-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+*.so
+
+# uv
+.uv/
+uv.lock
+
+# ragradar runtime — never commit user run data
+.ragradar/
+
+# environment
+.env
+*.env
+.env.*
+
+# IDE
+.vscode/
+.idea/
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# test output
+.pytest_cache/
+htmlcov/
+.coverage
+
+# example output
+examples/rag_pipeline/output/
+.claude/

ragradar-0.1.0/PKG-INFO

@@ -0,0 +1,142 @@
+Metadata-Version: 2.4
+Name: ragradar
+Version: 0.1.0
+Summary: RAG observability: capture, evaluate, and explain pipeline runs (umbrella package + analyst CLI)
+Project-URL: Homepage, https://github.com/pleokarthik/RAGRadar
+Project-URL: Repository, https://github.com/pleokarthik/RAGRadar
+Project-URL: Issues, https://github.com/pleokarthik/RAGRadar/issues
+Author-email: Leo Karthik Paramasivan <pleokarthik@gmail.com>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: click>=8.0
+Requires-Dist: ragradar-capture<0.2.0,>=0.1.0
+Requires-Dist: ragradar-core<0.2.0,>=0.1.0
+Requires-Dist: ragradar-evaluate<0.2.0,>=0.1.0
+Requires-Dist: rich>=13.0
+Provides-Extra: semantic
+Requires-Dist: sentence-transformers>=3.0; extra == 'semantic'
+Requires-Dist: sqlite-vec>=0.1; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# ragradar
+
+Analyst CLI for the ragradar observability system. Reads the local store at
+`~/.ragradar/runs.db` that `ragradar-capture` writes; browses sessions, searches
+runs, and explains exactly what went into a run's context window.
+
+```
+pip install ragradar
+```
+
+Runs are addressed as `sNrN` (session 2, run 3 → `s2r3`) — the id every
+capture call returns. Commands that take a `<target>` accept an exact
+id, nothing (= latest run), or a quoted text hint (searched; multiple
+matches show a pick list).
+
+## ragradar list — sessions and runs
+
+```bash
+ragradar list          # sessions, newest first
+ragradar list s2       # runs inside session 2
+```
+
+```
+                  Sessions
+| ID | Runs | Pipeline    | Created    | Title |
+|----+------+-------------+------------+-------|
+| s2 |    3 | rag_example | 2026-07-02 |       |
+| s1 |    1 | quickstart  | 2026-07-02 |       |
+```
+
+## ragradar find — search runs by query text
+
+```bash
+ragradar find "reranking"              # token match (FTS5)
+ragradar find "score scale" --exact    # phrase match
+ragradar find "RRF" --session s2       # scope to a session
+ragradar find "RRF" --pipeline rag_example
+ragradar find "RRF" --from 2026-07-01 --to 2026-07-02
+ragradar find "RRF" --today
+ragradar find --recent 5               # latest N runs, no hint
+```
+
+```
+        Search results (2)
+| Run   | Date       | Session | Query                            |
+|-------+------------+---------+----------------------------------|
+| s2 r3 | 2026-07-02 |         | what is RRF and how does it ...  |
+```
+
+## ragradar explain — the seven analysis factors
+
+```bash
+ragradar explain            # latest run
+ragradar explain s2r3       # specific run
+ragradar explain s2r3 --full
+ragradar explain s2r3 --html    # snapshot to ~/.ragradar/reports/s2r3.html
+```
+
+Renders every factor the captured data supports, silently skipping the
+rest: token usage, chunk scores, duplicates, truncation, dropped
+history, cache hits, and the final prompt. Runs scored by
+`ragradar-evaluate` also show an Evaluation Scores panel (risk score, policy
+violations, RAGAS metrics).
+
+```
++------------- Token Usage --------------+
+| Total: 1138/4096 (27.8%)               |
+|   Chunks:   625                        |
+|   Headroom: 196                        |
++----------------------------------------+
++------------- Duplicates ---------------+
+| 1 duplicate (25%): 1 window            |
++----------------------------------------+
+```
+
+## ragradar diff — compare two runs
+
+```bash
+ragradar diff s2r1 s2r3
+```
+
+Side-by-side query delta, chunks added/removed, per-chunk score deltas,
+token budget deltas, history and truncation changes. Ambiguous targets
+are rejected — use exact ids here.
+
+## ragradar budget — token waterfall only
+
+```bash
+ragradar budget s2r3
+```
+
+```
++------------- Token Usage --------------+
+| Total: 1138/4096 (27.8%)               |
+|   Chunks:   625   History: 13         |
+|   System:   500   Headroom: 196       |
++----------------------------------------+
+```
+
+## ragradar session rename
+
+```bash
+ragradar session rename s2 "RRF investigation"
+```
+
+```
+Session 2 renamed to "RRF investigation".
+```
+
+## Notes
+
+- Read-mostly by design: the only data this CLI writes is a session
+  title. (Opening the store may create/migrate `runs.db` via `ragradar-core`
+  — that's environment setup, not run data.)
+- No LLM anywhere in the navigation path; search is SQLite FTS5.
+- Optional semantic search: `pip install ragradar[semantic]`.

ragradar-0.1.0/README.md

@@ -0,0 +1,116 @@
+# ragradar
+
+Analyst CLI for the ragradar observability system. Reads the local store at
+`~/.ragradar/runs.db` that `ragradar-capture` writes; browses sessions, searches
+runs, and explains exactly what went into a run's context window.
+
+```
+pip install ragradar
+```
+
+Runs are addressed as `sNrN` (session 2, run 3 → `s2r3`) — the id every
+capture call returns. Commands that take a `<target>` accept an exact
+id, nothing (= latest run), or a quoted text hint (searched; multiple
+matches show a pick list).
+
+## ragradar list — sessions and runs
+
+```bash
+ragradar list          # sessions, newest first
+ragradar list s2       # runs inside session 2
+```
+
+```
+                  Sessions
+| ID | Runs | Pipeline    | Created    | Title |
+|----+------+-------------+------------+-------|
+| s2 |    3 | rag_example | 2026-07-02 |       |
+| s1 |    1 | quickstart  | 2026-07-02 |       |
+```
+
+## ragradar find — search runs by query text
+
+```bash
+ragradar find "reranking"              # token match (FTS5)
+ragradar find "score scale" --exact    # phrase match
+ragradar find "RRF" --session s2       # scope to a session
+ragradar find "RRF" --pipeline rag_example
+ragradar find "RRF" --from 2026-07-01 --to 2026-07-02
+ragradar find "RRF" --today
+ragradar find --recent 5               # latest N runs, no hint
+```
+
+```
+        Search results (2)
+| Run   | Date       | Session | Query                            |
+|-------+------------+---------+----------------------------------|
+| s2 r3 | 2026-07-02 |         | what is RRF and how does it ...  |
+```
+
+## ragradar explain — the seven analysis factors
+
+```bash
+ragradar explain            # latest run
+ragradar explain s2r3       # specific run
+ragradar explain s2r3 --full
+ragradar explain s2r3 --html    # snapshot to ~/.ragradar/reports/s2r3.html
+```
+
+Renders every factor the captured data supports, silently skipping the
+rest: token usage, chunk scores, duplicates, truncation, dropped
+history, cache hits, and the final prompt. Runs scored by
+`ragradar-evaluate` also show an Evaluation Scores panel (risk score, policy
+violations, RAGAS metrics).
+
+```
++------------- Token Usage --------------+
+| Total: 1138/4096 (27.8%)               |
+|   Chunks:   625                        |
+|   Headroom: 196                        |
++----------------------------------------+
++------------- Duplicates ---------------+
+| 1 duplicate (25%): 1 window            |
++----------------------------------------+
+```
+
+## ragradar diff — compare two runs
+
+```bash
+ragradar diff s2r1 s2r3
+```
+
+Side-by-side query delta, chunks added/removed, per-chunk score deltas,
+token budget deltas, history and truncation changes. Ambiguous targets
+are rejected — use exact ids here.
+
+## ragradar budget — token waterfall only
+
+```bash
+ragradar budget s2r3
+```
+
+```
++------------- Token Usage --------------+
+| Total: 1138/4096 (27.8%)               |
+|   Chunks:   625   History: 13         |
+|   System:   500   Headroom: 196       |
++----------------------------------------+
+```
+
+## ragradar session rename
+
+```bash
+ragradar session rename s2 "RRF investigation"
+```
+
+```
+Session 2 renamed to "RRF investigation".
+```
+
+## Notes
+
+- Read-mostly by design: the only data this CLI writes is a session
+  title. (Opening the store may create/migrate `runs.db` via `ragradar-core`
+  — that's environment setup, not run data.)
+- No LLM anywhere in the navigation path; search is SQLite FTS5.
+- Optional semantic search: `pip install ragradar[semantic]`.

ragradar-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "ragradar"
+version = "0.1.0"
+description = "RAG observability: capture, evaluate, and explain pipeline runs (umbrella package + analyst CLI)"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [
+    { name = "Leo Karthik Paramasivan", email = "pleokarthik@gmail.com" },
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "ragradar-core>=0.1.0,<0.2.0",
+    "ragradar-capture>=0.1.0,<0.2.0",
+    "ragradar-evaluate>=0.1.0,<0.2.0",
+    "rich>=13.0",
+    "click>=8.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/pleokarthik/RAGRadar"
+Repository = "https://github.com/pleokarthik/RAGRadar"
+Issues = "https://github.com/pleokarthik/RAGRadar/issues"
+
+[project.optional-dependencies]
+semantic = [
+    "sentence-transformers>=3.0",
+    "sqlite-vec>=0.1",
+]
+
+[project.scripts]
+ragradar = "ragradar.cli:main"
+
+[tool.uv.sources]
+ragradar-core = { workspace = true }
+ragradar-capture = { workspace = true }
+ragradar-evaluate = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ragradar"]

ragradar-0.1.0/src/ragradar/__init__.py

@@ -0,0 +1,74 @@
+"""ragradar — the single public import surface.
+
+Users only ever write ``import ragradar``: capture entry points
+(capture/start and the staged proxies), evaluation entry points
+(check/evaluate/available_metrics), and the schema dataclasses are all
+re-exported here. The underlying distributions (ragradar-core,
+ragradar-capture, ragradar-evaluate) stay separately installable so a
+production pipeline can depend on ragradar-capture alone without
+pulling the evaluation stack (scipy/ragas) — but importing their
+modules directly is an internal concern, not the public API.
+"""
+
+from ragradar_capture import (
+    Capture,
+    cache,
+    capture,
+    chunks,
+    commit,
+    context,
+    history,
+    response,
+    set_strict,
+    start,
+    tool_call,
+)
+from ragradar_core.schema import (
+    CacheEvent,
+    ChunkRecord,
+    RunRecord,
+    TokenBudget,
+    TokenUsage,
+    ToolCallRecord,
+    Turn,
+)
+from ragradar_evaluate import (
+    CheckResult,
+    EvalResult,
+    InputQualityPolicy,
+    MetricInfo,
+    available_metrics,
+    check,
+    evaluate,
+)
+
+__all__ = [
+    # Capture
+    "Capture",
+    "start",
+    "capture",
+    "set_strict",
+    "chunks",
+    "context",
+    "history",
+    "response",
+    "cache",
+    "tool_call",
+    "commit",
+    # Evaluation
+    "check",
+    "evaluate",
+    "available_metrics",
+    "CheckResult",
+    "EvalResult",
+    "MetricInfo",
+    "InputQualityPolicy",
+    # Schema dataclasses (advanced path; primitives coerce everywhere)
+    "ChunkRecord",
+    "TokenBudget",
+    "TokenUsage",
+    "Turn",
+    "CacheEvent",
+    "ToolCallRecord",
+    "RunRecord",
+]

ragradar-0.1.0/src/ragradar/cli.py

@@ -0,0 +1,231 @@
+import re
+from datetime import date
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from ragradar import store
+from ragradar.explain import loader
+from ragradar.explain.renderer import html as html_renderer
+from ragradar.explain.renderer import terminal as terminal_renderer
+
+console = Console()
+
+_SESSION_RE = re.compile(r"^s(\d+)$", re.IGNORECASE)
+
+
+def _parse_session_id(value: str) -> int:
+    m = _SESSION_RE.match(value)
+    if m:
+        return int(m.group(1))
+    return int(value)
+
+
+def _disambiguate(results: list[dict]) -> dict | None:
+    console.print("\n  [bold]Multiple matches:[/bold]\n")
+    for i, r in enumerate(results, 1):
+        title = r.get("session_title") or ""
+        query_preview = r["query"][:60]
+        console.print(
+            f"  {i}   s{r['session_id']} r{r['run_seq']}   "
+            f"{r['created_at'][:10]}   {title}   "
+            f'— "{query_preview}"'
+        )
+    console.print()
+    try:
+        choice = click.prompt(
+            "  Pick (number) or press Enter to cancel",
+            default="",
+            show_default=False,
+        )
+        if not choice:
+            return None
+        idx = int(choice) - 1
+        if 0 <= idx < len(results):
+            r = results[idx]
+            return store.get_run(r["session_id"], r["run_seq"])
+    except (ValueError, KeyboardInterrupt, EOFError):
+        pass
+    return None
+
+
+def _resolve_and_load(target: str | None = None):
+    result = store.resolve_target(target)
+    if result is None:
+        console.print("No runs found.")
+        return None, None
+    if isinstance(result, list):
+        run_row = _disambiguate(result)
+        if run_row is None:
+            return None, None
+    else:
+        run_row = result
+    record = loader.load_run_record(run_row)
+    return run_row, record
+
+
+@click.group()
+def main():
+    """ragradar — analyst CLI for the ragradar observability system."""
+
+
+@main.command("list")
+@click.argument("session_id", required=False)
+def list_cmd(session_id):
+    """List sessions, or runs within a session."""
+    if session_id is not None:
+        sid = _parse_session_id(session_id)
+        runs = store.list_runs(sid)
+        if not runs:
+            console.print(f"No runs found in session {sid}.")
+            return
+        tbl = Table(title=f"Session {sid} — Runs")
+        tbl.add_column("Run", style="cyan")
+        tbl.add_column("Date")
+        tbl.add_column("Query")
+        for r in runs:
+            tbl.add_row(
+                f"s{r['session_id']} r{r['run_seq']}",
+                r["created_at"][:10],
+                r["query"][:80],
+            )
+        console.print(tbl)
+    else:
+        sessions = store.list_sessions()
+        if not sessions:
+            console.print("No sessions found.")
+            return
+        tbl = Table(title="Sessions")
+        tbl.add_column("ID", style="cyan")
+        tbl.add_column("Runs", justify="right")
+        tbl.add_column("Pipeline")
+        tbl.add_column("Created")
+        tbl.add_column("Title")
+        for s in sessions:
+            tbl.add_row(
+                f"s{s['session_id']}",
+                str(s["run_count"]),
+                s["pipeline"] or "",
+                s["created_at"][:10],
+                s["title"] or "",
+            )
+        console.print(tbl)
+
+
+@main.command()
+@click.argument("hint", required=False)
+@click.option("--exact", is_flag=True)
+@click.option("--from", "from_dt", default=None)
+@click.option("--to", "to_dt", default=None)
+@click.option("--today", is_flag=True)
+@click.option("--session", "session_filter", default=None)
+@click.option("--pipeline", default=None)
+@click.option("--recent", default=None, type=int)
+def find(hint, exact, from_dt, to_dt, today, session_filter, pipeline, recent):
+    """Search runs by query text."""
+    if today:
+        today_str = date.today().isoformat()
+        if from_dt is None:
+            from_dt = today_str
+        if to_dt is None:
+            to_dt = today_str + "T23:59:59.999999Z"
+
+    sid = None
+    if session_filter is not None:
+        sid = _parse_session_id(session_filter)
+
+    results = store.search_runs(
+        hint=hint,
+        exact=exact,
+        session_id=sid,
+        pipeline=pipeline,
+        from_dt=from_dt,
+        to_dt=to_dt,
+        recent_n=recent,
+    )
+
+    if not results:
+        console.print("No matching runs found.")
+        return
+
+    tbl = Table(title=f"Search results ({len(results)})")
+    tbl.add_column("Run", style="cyan")
+    tbl.add_column("Date")
+    tbl.add_column("Session")
+    tbl.add_column("Query")
+    for r in results:
+        tbl.add_row(
+            f"s{r['session_id']} r{r['run_seq']}",
+            r["created_at"][:10],
+            r.get("session_title") or "",
+            r["query"][:80],
+        )
+    console.print(tbl)
+
+
+@main.command()
+@click.argument("target", required=False)
+@click.option("--full", is_flag=True)
+@click.option("--html", "to_html", is_flag=True)
+def explain(target, full, to_html):
+    """Explain a run — all analysis factors."""
+    run_row, record = _resolve_and_load(target)
+    if record is None:
+        return
+
+    if to_html:
+        run_id = f"s{run_row['session_id']}r{run_row['run_seq']}"
+        path = html_renderer.render(record, run_id)
+        console.print(f"Report written to {path}")
+    else:
+        terminal_renderer.render(record, full=full, run_row=run_row)
+
+
+@main.command()
+@click.argument("target_a")
+@click.argument("target_b")
+def diff(target_a, target_b):
+    """Compare two runs side by side."""
+    row_a = store.resolve_target(target_a)
+    row_b = store.resolve_target(target_b)
+
+    if row_a is None or row_b is None:
+        console.print("Could not resolve both targets.")
+        return
+    if isinstance(row_a, list) or isinstance(row_b, list):
+        console.print("Ambiguous target — use exact run ID (e.g. s2r3).")
+        return
+
+    rec_a = loader.load_run_record(row_a)
+    rec_b = loader.load_run_record(row_b)
+
+    id_a = f"s{row_a['session_id']}r{row_a['run_seq']}"
+    id_b = f"s{row_b['session_id']}r{row_b['run_seq']}"
+
+    terminal_renderer.render_diff(rec_a, rec_b, id_a, id_b)
+
+
+@main.command()
+@click.argument("target")
+def budget(target):
+    """Token waterfall only."""
+    run_row, record = _resolve_and_load(target)
+    if record is None:
+        return
+    terminal_renderer.render_budget(record)
+
+
+@main.group()
+def session():
+    """Session management commands."""
+
+
+@session.command()
+@click.argument("session_id")
+@click.argument("title")
+def rename(session_id, title):
+    """Rename a session."""
+    sid = _parse_session_id(session_id)
+    store.rename_session(sid, title)
+    console.print(f'Session {sid} renamed to "{title}".')

ragradar-0.1.0/src/ragradar/explain/analyzers/cache.py

@@ -0,0 +1,19 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.cache_events:
+        return None
+
+    hits = [e for e in record.cache_events if e.hit]
+    misses = [e for e in record.cache_events if not e.hit]
+    total = len(record.cache_events)
+
+    return {
+        "total_events": total,
+        "hits": len(hits),
+        "misses": len(misses),
+        "hit_ratio": len(hits) / total if total else 0.0,
+        "hit_chunks": [e.chunk_id for e in hits],
+        "miss_chunks": [e.chunk_id for e in misses],
+    }