ragradar 0.1.0__py3-none-any.whl

ragradar/__init__.py

@@ -0,0 +1,74 @@
+"""ragradar — the single public import surface.
+
+Users only ever write ``import ragradar``: capture entry points
+(capture/start and the staged proxies), evaluation entry points
+(check/evaluate/available_metrics), and the schema dataclasses are all
+re-exported here. The underlying distributions (ragradar-core,
+ragradar-capture, ragradar-evaluate) stay separately installable so a
+production pipeline can depend on ragradar-capture alone without
+pulling the evaluation stack (scipy/ragas) — but importing their
+modules directly is an internal concern, not the public API.
+"""
+
+from ragradar_capture import (
+    Capture,
+    cache,
+    capture,
+    chunks,
+    commit,
+    context,
+    history,
+    response,
+    set_strict,
+    start,
+    tool_call,
+)
+from ragradar_core.schema import (
+    CacheEvent,
+    ChunkRecord,
+    RunRecord,
+    TokenBudget,
+    TokenUsage,
+    ToolCallRecord,
+    Turn,
+)
+from ragradar_evaluate import (
+    CheckResult,
+    EvalResult,
+    InputQualityPolicy,
+    MetricInfo,
+    available_metrics,
+    check,
+    evaluate,
+)
+
+__all__ = [
+    # Capture
+    "Capture",
+    "start",
+    "capture",
+    "set_strict",
+    "chunks",
+    "context",
+    "history",
+    "response",
+    "cache",
+    "tool_call",
+    "commit",
+    # Evaluation
+    "check",
+    "evaluate",
+    "available_metrics",
+    "CheckResult",
+    "EvalResult",
+    "MetricInfo",
+    "InputQualityPolicy",
+    # Schema dataclasses (advanced path; primitives coerce everywhere)
+    "ChunkRecord",
+    "TokenBudget",
+    "TokenUsage",
+    "Turn",
+    "CacheEvent",
+    "ToolCallRecord",
+    "RunRecord",
+]

ragradar/cli.py

@@ -0,0 +1,231 @@
+import re
+from datetime import date
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from ragradar import store
+from ragradar.explain import loader
+from ragradar.explain.renderer import html as html_renderer
+from ragradar.explain.renderer import terminal as terminal_renderer
+
+console = Console()
+
+_SESSION_RE = re.compile(r"^s(\d+)$", re.IGNORECASE)
+
+
+def _parse_session_id(value: str) -> int:
+    m = _SESSION_RE.match(value)
+    if m:
+        return int(m.group(1))
+    return int(value)
+
+
+def _disambiguate(results: list[dict]) -> dict | None:
+    console.print("\n  [bold]Multiple matches:[/bold]\n")
+    for i, r in enumerate(results, 1):
+        title = r.get("session_title") or ""
+        query_preview = r["query"][:60]
+        console.print(
+            f"  {i}   s{r['session_id']} r{r['run_seq']}   "
+            f"{r['created_at'][:10]}   {title}   "
+            f'— "{query_preview}"'
+        )
+    console.print()
+    try:
+        choice = click.prompt(
+            "  Pick (number) or press Enter to cancel",
+            default="",
+            show_default=False,
+        )
+        if not choice:
+            return None
+        idx = int(choice) - 1
+        if 0 <= idx < len(results):
+            r = results[idx]
+            return store.get_run(r["session_id"], r["run_seq"])
+    except (ValueError, KeyboardInterrupt, EOFError):
+        pass
+    return None
+
+
+def _resolve_and_load(target: str | None = None):
+    result = store.resolve_target(target)
+    if result is None:
+        console.print("No runs found.")
+        return None, None
+    if isinstance(result, list):
+        run_row = _disambiguate(result)
+        if run_row is None:
+            return None, None
+    else:
+        run_row = result
+    record = loader.load_run_record(run_row)
+    return run_row, record
+
+
+@click.group()
+def main():
+    """ragradar — analyst CLI for the ragradar observability system."""
+
+
+@main.command("list")
+@click.argument("session_id", required=False)
+def list_cmd(session_id):
+    """List sessions, or runs within a session."""
+    if session_id is not None:
+        sid = _parse_session_id(session_id)
+        runs = store.list_runs(sid)
+        if not runs:
+            console.print(f"No runs found in session {sid}.")
+            return
+        tbl = Table(title=f"Session {sid} — Runs")
+        tbl.add_column("Run", style="cyan")
+        tbl.add_column("Date")
+        tbl.add_column("Query")
+        for r in runs:
+            tbl.add_row(
+                f"s{r['session_id']} r{r['run_seq']}",
+                r["created_at"][:10],
+                r["query"][:80],
+            )
+        console.print(tbl)
+    else:
+        sessions = store.list_sessions()
+        if not sessions:
+            console.print("No sessions found.")
+            return
+        tbl = Table(title="Sessions")
+        tbl.add_column("ID", style="cyan")
+        tbl.add_column("Runs", justify="right")
+        tbl.add_column("Pipeline")
+        tbl.add_column("Created")
+        tbl.add_column("Title")
+        for s in sessions:
+            tbl.add_row(
+                f"s{s['session_id']}",
+                str(s["run_count"]),
+                s["pipeline"] or "",
+                s["created_at"][:10],
+                s["title"] or "",
+            )
+        console.print(tbl)
+
+
+@main.command()
+@click.argument("hint", required=False)
+@click.option("--exact", is_flag=True)
+@click.option("--from", "from_dt", default=None)
+@click.option("--to", "to_dt", default=None)
+@click.option("--today", is_flag=True)
+@click.option("--session", "session_filter", default=None)
+@click.option("--pipeline", default=None)
+@click.option("--recent", default=None, type=int)
+def find(hint, exact, from_dt, to_dt, today, session_filter, pipeline, recent):
+    """Search runs by query text."""
+    if today:
+        today_str = date.today().isoformat()
+        if from_dt is None:
+            from_dt = today_str
+        if to_dt is None:
+            to_dt = today_str + "T23:59:59.999999Z"
+
+    sid = None
+    if session_filter is not None:
+        sid = _parse_session_id(session_filter)
+
+    results = store.search_runs(
+        hint=hint,
+        exact=exact,
+        session_id=sid,
+        pipeline=pipeline,
+        from_dt=from_dt,
+        to_dt=to_dt,
+        recent_n=recent,
+    )
+
+    if not results:
+        console.print("No matching runs found.")
+        return
+
+    tbl = Table(title=f"Search results ({len(results)})")
+    tbl.add_column("Run", style="cyan")
+    tbl.add_column("Date")
+    tbl.add_column("Session")
+    tbl.add_column("Query")
+    for r in results:
+        tbl.add_row(
+            f"s{r['session_id']} r{r['run_seq']}",
+            r["created_at"][:10],
+            r.get("session_title") or "",
+            r["query"][:80],
+        )
+    console.print(tbl)
+
+
+@main.command()
+@click.argument("target", required=False)
+@click.option("--full", is_flag=True)
+@click.option("--html", "to_html", is_flag=True)
+def explain(target, full, to_html):
+    """Explain a run — all analysis factors."""
+    run_row, record = _resolve_and_load(target)
+    if record is None:
+        return
+
+    if to_html:
+        run_id = f"s{run_row['session_id']}r{run_row['run_seq']}"
+        path = html_renderer.render(record, run_id)
+        console.print(f"Report written to {path}")
+    else:
+        terminal_renderer.render(record, full=full, run_row=run_row)
+
+
+@main.command()
+@click.argument("target_a")
+@click.argument("target_b")
+def diff(target_a, target_b):
+    """Compare two runs side by side."""
+    row_a = store.resolve_target(target_a)
+    row_b = store.resolve_target(target_b)
+
+    if row_a is None or row_b is None:
+        console.print("Could not resolve both targets.")
+        return
+    if isinstance(row_a, list) or isinstance(row_b, list):
+        console.print("Ambiguous target — use exact run ID (e.g. s2r3).")
+        return
+
+    rec_a = loader.load_run_record(row_a)
+    rec_b = loader.load_run_record(row_b)
+
+    id_a = f"s{row_a['session_id']}r{row_a['run_seq']}"
+    id_b = f"s{row_b['session_id']}r{row_b['run_seq']}"
+
+    terminal_renderer.render_diff(rec_a, rec_b, id_a, id_b)
+
+
+@main.command()
+@click.argument("target")
+def budget(target):
+    """Token waterfall only."""
+    run_row, record = _resolve_and_load(target)
+    if record is None:
+        return
+    terminal_renderer.render_budget(record)
+
+
+@main.group()
+def session():
+    """Session management commands."""
+
+
+@session.command()
+@click.argument("session_id")
+@click.argument("title")
+def rename(session_id, title):
+    """Rename a session."""
+    sid = _parse_session_id(session_id)
+    store.rename_session(sid, title)
+    console.print(f'Session {sid} renamed to "{title}".')

ragradar/explain/analyzers/cache.py

@@ -0,0 +1,19 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.cache_events:
+        return None
+
+    hits = [e for e in record.cache_events if e.hit]
+    misses = [e for e in record.cache_events if not e.hit]
+    total = len(record.cache_events)
+
+    return {
+        "total_events": total,
+        "hits": len(hits),
+        "misses": len(misses),
+        "hit_ratio": len(hits) / total if total else 0.0,
+        "hit_chunks": [e.chunk_id for e in hits],
+        "miss_chunks": [e.chunk_id for e in misses],
+    }

ragradar/explain/analyzers/duplicates.py

@@ -0,0 +1,49 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.chunks:
+        return None
+
+    # PATH DUP: same chunk_id appears multiple times with different retrieval_path
+    chunk_paths: dict[str, list[str]] = {}
+    for c in record.chunks:
+        chunk_paths.setdefault(c.chunk_id, [])
+        if c.retrieval_path:
+            chunk_paths[c.chunk_id].append(c.retrieval_path)
+
+    path_dups = [
+        {"chunk_id": cid, "paths": paths} for cid, paths in chunk_paths.items() if len(paths) > 1
+    ]
+
+    # WINDOW DUP: same source_doc_id, overlapping content (substring check)
+    by_source: dict[str, list] = {}
+    for c in record.chunks:
+        by_source.setdefault(c.source_doc_id, []).append(c)
+
+    window_dups = []
+    for source_id, chunks in by_source.items():
+        if len(chunks) < 2:
+            continue
+        dup_ids: set[str] = set()
+        for i, a in enumerate(chunks):
+            for b in chunks[i + 1 :]:
+                if a.content in b.content or b.content in a.content:
+                    dup_ids.add(a.chunk_id)
+                    dup_ids.add(b.chunk_id)
+        if dup_ids:
+            window_dups.append({"chunk_ids": sorted(dup_ids), "source_doc_id": source_id})
+
+    dup_chunk_ids: set[str] = set()
+    for d in path_dups:
+        dup_chunk_ids.add(d["chunk_id"])
+    for d in window_dups:
+        dup_chunk_ids.update(d["chunk_ids"])
+
+    total = len(record.chunks)
+    return {
+        "path_dups": path_dups,
+        "window_dups": window_dups,
+        "semantic_dups": [],
+        "duplicate_ratio": len(dup_chunk_ids) / total if total else 0.0,
+    }

ragradar/explain/analyzers/history.py

@@ -0,0 +1,25 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.history_pre and not record.history_post:
+        return None
+
+    pre = record.history_pre or []
+    post = record.history_post or []
+
+    pre_vals = [t.tokens for t in pre if t.tokens is not None]
+    post_vals = [t.tokens for t in post if t.tokens is not None]
+
+    post_contents = {(t.role, t.content) for t in post}
+    dropped = [t for t in pre if (t.role, t.content) not in post_contents]
+
+    return {
+        "pre_turn_count": len(pre),
+        "post_turn_count": len(post),
+        "dropped_turn_count": len(dropped),
+        "eviction_reason": record.eviction_reason,
+        "dropped_turns": dropped,
+        "pre_tokens": sum(pre_vals) if pre_vals else None,
+        "post_tokens": sum(post_vals) if post_vals else None,
+    }

ragradar/explain/analyzers/scores.py

@@ -0,0 +1,38 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.chunks:
+        return None
+
+    retrieval_scores = [c.retrieval_score for c in record.chunks if c.retrieval_score is not None]
+    rerank_scores = [c.rerank_score for c in record.chunks if c.rerank_score is not None]
+
+    if not retrieval_scores and not rerank_scores:
+        return None
+
+    top_retrieval = max(retrieval_scores) if retrieval_scores else None
+    bottom_retrieval = min(retrieval_scores) if retrieval_scores else None
+    top_rerank = max(rerank_scores) if rerank_scores else None
+    bottom_rerank = min(rerank_scores) if rerank_scores else None
+
+    rerank_delta = None
+    if retrieval_scores and rerank_scores:
+        mean_rerank = sum(rerank_scores) / len(rerank_scores)
+        mean_retrieval = sum(retrieval_scores) / len(retrieval_scores)
+        rerank_delta = round(mean_rerank - mean_retrieval, 4)
+
+    total = len(record.chunks)
+    low_rerank = [s for s in rerank_scores if s < 0.5]
+    low_score_ratio = round(len(low_rerank) / total, 4) if total else 0.0
+
+    return {
+        "retrieval_scores": retrieval_scores,
+        "rerank_scores": rerank_scores,
+        "top_retrieval": top_retrieval,
+        "top_rerank": top_rerank,
+        "bottom_retrieval": bottom_retrieval,
+        "bottom_rerank": bottom_rerank,
+        "rerank_delta": rerank_delta,
+        "low_score_ratio": low_score_ratio,
+    }

ragradar/explain/analyzers/tokens.py

@@ -0,0 +1,36 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.chunks and not record.final_prompt:
+        return None
+
+    chunks_tokens = sum(c.token_count for c in (record.chunks or []))
+
+    history_tokens = sum(t.tokens or 0 for t in (record.history_post or []))
+
+    system_tokens = 0
+    headroom = 0
+    model_limit = None
+    if record.token_budget:
+        system_tokens = record.token_budget.system_allocated
+        headroom = record.token_budget.headroom
+        model_limit = record.token_budget.total_limit
+
+    total = chunks_tokens + history_tokens + system_tokens
+    utilisation = (total / model_limit * 100) if model_limit else 0.0
+
+    per_chunk = [
+        {"chunk_id": c.chunk_id, "token_count": c.token_count} for c in (record.chunks or [])
+    ]
+
+    return {
+        "total_tokens": total,
+        "chunks_tokens": chunks_tokens,
+        "history_tokens": history_tokens,
+        "system_tokens": system_tokens,
+        "headroom": headroom,
+        "model_limit": model_limit,
+        "utilisation_pct": round(utilisation, 1),
+        "per_chunk": per_chunk,
+    }

ragradar/explain/analyzers/truncation.py

@@ -0,0 +1,32 @@
+from ragradar_core.schema import RunRecord
+
+
+def analyze(record: RunRecord) -> dict | None:
+    if not record.chunks:
+        return None
+
+    truncated = [c for c in record.chunks if c.truncated]
+    high_score = [
+        c for c in truncated if (c.retrieval_score or 0) > 0.7 or (c.rerank_score or 0) > 0.7
+    ]
+
+    if not truncated:
+        severity = "none"
+    elif high_score:
+        severity = "high"
+    else:
+        severity = "low"
+
+    return {
+        "truncated_count": len(truncated),
+        "truncated_chunks": [
+            {
+                "chunk_id": c.chunk_id,
+                "score": c.retrieval_score,
+                "rerank_score": c.rerank_score,
+            }
+            for c in truncated
+        ],
+        "high_score_truncations": len(high_score),
+        "severity": severity,
+    }

ragradar/explain/loader.py

@@ -0,0 +1,8 @@
+import json
+
+from ragradar_core.schema import RunRecord
+
+
+def load_run_record(run_row: dict) -> RunRecord:
+    data = json.loads(run_row["run_data"])
+    return RunRecord.from_json(data)

ragradar/explain/renderer/html.py

@@ -0,0 +1,152 @@
+from pathlib import Path
+
+from ragradar_core import store as _core_store
+from ragradar_core.schema import RunRecord
+
+from ragradar.explain.analyzers import (
+    cache as cache_mod,
+)
+from ragradar.explain.analyzers import (
+    duplicates as duplicates_mod,
+)
+from ragradar.explain.analyzers import (
+    history as history_mod,
+)
+from ragradar.explain.analyzers import (
+    scores as scores_mod,
+)
+from ragradar.explain.analyzers import (
+    tokens as tokens_mod,
+)
+from ragradar.explain.analyzers import (
+    truncation as truncation_mod,
+)
+
+
+def _esc(text: str) -> str:
+    return (
+        text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+    )
+
+
+def _section(title: str, content: str) -> str:
+    return (
+        f"<details open>\n  <summary>{_esc(title)}</summary>\n"
+        f"  <pre>{_esc(content)}</pre>\n</details>\n"
+    )
+
+
+def render(record: RunRecord, run_id: str) -> Path:
+    reports_dir = _core_store._ragradar_dir() / "reports"
+    reports_dir.mkdir(parents=True, exist_ok=True)
+    out = reports_dir / f"{run_id}.html"
+
+    sections = ""
+
+    tok = tokens_mod.analyze(record)
+    if tok:
+        body = f"Total: {tok['total_tokens']}"
+        if tok["model_limit"]:
+            body += f" / {tok['model_limit']} ({tok['utilisation_pct']}%)"
+        body += f"\nChunks:   {tok['chunks_tokens']}"
+        body += f"\nHistory:  {tok['history_tokens']}"
+        body += f"\nSystem:   {tok['system_tokens']}"
+        body += f"\nHeadroom: {tok['headroom']}"
+        if tok["per_chunk"]:
+            body += "\n\nPer chunk:"
+            for pc in tok["per_chunk"]:
+                body += f"\n  {pc['chunk_id']}: {pc['token_count']}"
+        sections += _section("Token Usage", body)
+
+    sc = scores_mod.analyze(record)
+    if sc:
+        body = ""
+        if sc["retrieval_scores"]:
+            body += f"Retrieval: {sc['bottom_retrieval']:.2f}–{sc['top_retrieval']:.2f}\n"
+        if sc["rerank_scores"]:
+            body += f"Rerank:    {sc['bottom_rerank']:.2f}–{sc['top_rerank']:.2f}\n"
+        if sc["rerank_delta"] is not None:
+            body += f"Rerank delta: {sc['rerank_delta']:+.4f}\n"
+        body += f"Low-score ratio: {sc['low_score_ratio']:.0%}"
+        sections += _section("Chunk Scores", body)
+
+    dup = duplicates_mod.analyze(record)
+    if dup:
+        body = f"Path dups: {len(dup['path_dups'])}\n"
+        body += f"Window dups: {len(dup['window_dups'])}\n"
+        body += f"Duplicate ratio: {dup['duplicate_ratio']:.0%}"
+        for d in dup["path_dups"]:
+            body += f"\n  [PATH DUP] {d['chunk_id']} via {', '.join(d['paths'])}"
+        for d in dup["window_dups"]:
+            ids = ", ".join(d["chunk_ids"])
+            body += f"\n  [WINDOW DUP] {ids} (source: {d['source_doc_id']})"
+        sections += _section("Duplicate Chunks", body)
+
+    tr = truncation_mod.analyze(record)
+    if tr:
+        body = f"Truncated: {tr['truncated_count']} chunks\n"
+        body += f"High-score truncations: {tr['high_score_truncations']}\n"
+        body += f"Severity: {tr['severity']}"
+        for tc in tr["truncated_chunks"]:
+            body += f"\n  {tc['chunk_id']}: retrieval={tc['score']}, rerank={tc['rerank_score']}"
+        sections += _section("Truncation", body)
+
+    hist = history_mod.analyze(record)
+    if hist:
+        body = (
+            f"Turns: {hist['pre_turn_count']} -> {hist['post_turn_count']} "
+            f"({hist['dropped_turn_count']} dropped)\n"
+        )
+        if hist["eviction_reason"]:
+            body += f"Reason: {hist['eviction_reason']}\n"
+        if hist["pre_tokens"] is not None:
+            body += f"Pre tokens: {hist['pre_tokens']}\n"
+        if hist["post_tokens"] is not None:
+            body += f"Post tokens: {hist['post_tokens']}\n"
+        for t in hist["dropped_turns"]:
+            body += f"\n  [{t.role}] {t.content}"
+        sections += _section("Dropped History", body)
+
+    ca = cache_mod.analyze(record)
+    if ca:
+        body = f"Hits: {ca['hits']}/{ca['total_events']} ({ca['hit_ratio']:.0%})\n"
+        if ca["hit_chunks"]:
+            body += f"Hit chunks: {', '.join(ca['hit_chunks'])}\n"
+        if ca["miss_chunks"]:
+            body += f"Miss chunks: {', '.join(ca['miss_chunks'])}"
+        sections += _section("Cache Hits", body)
+
+    if record.final_prompt:
+        sections += _section("Final Prompt", record.final_prompt)
+
+    model_line = ""
+    if record.model:
+        model_line = f"<p><b>Model:</b> {_esc(record.model)}</p>"
+
+    html = f"""<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>ragradar report — {_esc(run_id)}</title>
+<style>
+body {{ font-family: system-ui, monospace; max-width: 900px; margin: 2em auto; padding: 0 1em; }}
+details {{ margin: 1em 0; border: 1px solid #ccc; border-radius: 4px; padding: 0.5em 1em; }}
+summary {{ cursor: pointer; font-weight: bold; padding: 0.3em 0; }}
+pre {{ background: #f5f5f5; padding: 1em; overflow-x: auto; white-space: pre-wrap; }}
+h1 {{ color: #333; }}
+.meta {{ color: #666; margin-bottom: 2em; }}
+</style>
+</head>
+<body>
+<h1>ragradar report — {_esc(run_id)}</h1>
+<div class="meta">
+<p><b>Query:</b> {_esc(record.query)}</p>
+<p><b>Response:</b> {_esc(record.response[:500])}</p>
+{model_line}
+</div>
+{sections}
+</body>
+</html>"""
+
+    out.write_text(html, encoding="utf-8")
+    return out