diffprompt 0.1.0__py3-none-any.whl

diffprompt/models/cascade.py

@@ -0,0 +1,144 @@
+"""
+Cascade model client.
+Tries Ollama first (local, free), falls back to Groq.
+
+Key design decisions:
+- Exponential backoff on transient failures (not just silently returning None)
+- call_groq_only() used by judge escalation so it doesn't re-try Ollama
+"""
+from __future__ import annotations
+import asyncio
+import httpx
+import os
+from typing import Optional
+
+
+OLLAMA_BASE = "http://localhost:11434"
+GROQ_BASE   = "https://api.groq.com/openai/v1"
+
+_MAX_RETRIES = 2
+_RETRY_DELAY = 1.0  # seconds
+
+
+async def call_ollama(
+    model: str,
+    prompt: str,
+    system: Optional[str] = None,
+) -> Optional[str]:
+    """Call a local Ollama model. Returns None if Ollama is not running."""
+    messages = []
+    if system:
+        messages.append({"role": "system", "content": system})
+    messages.append({"role": "user", "content": prompt})
+
+    for attempt in range(_MAX_RETRIES):
+        try:
+            async with httpx.AsyncClient(timeout=60.0) as client:
+                r = await client.post(
+                    f"{OLLAMA_BASE}/api/chat",
+                    json={"model": model, "messages": messages, "stream": False},
+                )
+                r.raise_for_status()
+                return r.json()["message"]["content"]
+        except httpx.ConnectError:
+            return None  # Ollama not running — don't retry
+        except Exception:
+            if attempt < _MAX_RETRIES - 1:
+                await asyncio.sleep(_RETRY_DELAY * (attempt + 1))
+    return None
+
+async def call_groq(
+    model: str,
+    prompt: str,
+    system: Optional[str] = None,
+) -> Optional[str]:
+    """Call Groq API with loud error logging and backoff."""
+    api_key = os.getenv("GROQ_API_KEY")
+    if not api_key:
+        print("\n[DEBUG] The code cannot find your GROQ_API_KEY. VS Code is not loading it.")
+        return None
+
+    messages = []
+    if system:
+        messages.append({"role": "system", "content": system})
+    messages.append({"role": "user", "content": prompt})
+
+    _MAX_RETRIES = 5  # Increased from 2 to handle rate limits better
+
+    for attempt in range(_MAX_RETRIES):
+        try:
+            # We keep your trust_env=False fix here
+            async with httpx.AsyncClient(timeout=60.0, trust_env=False) as client:
+                r = await client.post(
+                    f"{GROQ_BASE}/chat/completions",
+                    headers={"Authorization": f"Bearer {api_key}"},
+                    json={"model": model, "messages": messages, "max_tokens": 2000},
+                )
+                r.raise_for_status()
+                return r.json()["choices"][0]["message"]["content"]
+                
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 401:
+                print(f"\n[DEBUG] HTTP 401: Your Groq API key is rejected.")
+                return None
+            elif e.response.status_code in (429, 503):
+                # We read the exact header Groq sends to know how long to wait
+                wait_time = float(e.response.headers.get("retry-after", _RETRY_DELAY * (2 ** attempt)))
+                print(f"\n[DEBUG] Rate limit hit. Waiting {wait_time}s...")
+                await asyncio.sleep(wait_time)
+            else:
+                print(f"\n[DEBUG] Groq API returned HTTP {e.response.status_code}: {e.response.text}")
+                return None
+        except httpx.ConnectError:
+             print("\n[DEBUG] ConnectError: Still blocked from the internet. Are you behind a corporate firewall?")
+             return None
+        except Exception as e:
+            print(f"\n[DEBUG] Unexpected error: {str(e)}")
+            if attempt < _MAX_RETRIES - 1:
+                await asyncio.sleep(_RETRY_DELAY)
+                
+    return None
+
+
+async def call_cascade(
+    prompt: str,
+    system: Optional[str] = None,
+    local_model: str = "qwen2.5:7b",
+    groq_model: str = "llama-3.1-8b-instant",
+    local_only: bool = False,
+) -> tuple[str, str]:
+    """
+    Try models in order: Ollama → Groq → error.
+    Returns (output, model_used).
+    """
+    result = await call_ollama(local_model, prompt, system)
+    if result:
+        return result, f"local/{local_model}"
+
+    if local_only:
+        raise RuntimeError(
+            "Ollama unavailable and --local-only is set. Is Ollama running?"
+        )
+
+    result = await call_groq(groq_model, prompt, system)
+    if result:
+        return result, f"groq/{groq_model}"
+
+    raise RuntimeError(
+        "All models failed. Check: is Ollama running? Is GROQ_API_KEY set?\n"
+        "  → Install Ollama: https://ollama.ai\n"
+        "  → Get free Groq key: https://console.groq.com"
+    )
+
+
+async def call_groq_only(
+    prompt: str,
+    system: Optional[str] = None,
+    groq_model: str = "llama-3.3-70b-versatile",
+) -> Optional[str]:
+    """
+    Directly calls Groq, skipping Ollama.
+    Used by judge escalation — we explicitly want the larger cloud model,
+    not a retry of the same local model that gave a low-confidence result.
+    """
+    return await call_groq(groq_model, prompt, system)

diffprompt/output/exporter.py

@@ -0,0 +1,227 @@
+"""
+HTML export for diffprompt reports.
+Self-contained single-file output, no external dependencies.
+"""
+from __future__ import annotations
+from collections import defaultdict
+from diffprompt.models import DiffReport, Verdict
+
+_VERDICT_COLOR = {
+    Verdict.IMPROVEMENT: "#22c55e",
+    Verdict.REGRESSION:  "#ef4444",
+    Verdict.NEUTRAL:     "#94a3b8",
+}
+_VERDICT_LABEL = {
+    Verdict.IMPROVEMENT: "SHIP IT",
+    Verdict.NEUTRAL:     "CONDITIONAL",
+    Verdict.REGRESSION:  "DO NOT SHIP",
+}
+
+
+def render_html(report: DiffReport) -> str:
+    score         = report.regression_score
+    score_color   = "#22c55e" if score >= 80 else "#f59e0b" if score >= 60 else "#ef4444"
+    verdict_color = _VERDICT_COLOR[report.verdict]
+    verdict_label = _VERDICT_LABEL[report.verdict]
+
+    slices_html   = _render_slices(report)
+    examples_html = _render_examples(report)
+    clusters_html = _render_clusters(report)
+
+    return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>diffprompt report</title>
+<style>
+  * {{ box-sizing: border-box; margin: 0; padding: 0; }}
+  body {{ font-family: 'JetBrains Mono', 'Fira Code', ui-monospace, monospace; background: #0f172a; color: #e2e8f0; padding: 2rem; line-height: 1.6; max-width: 960px; margin: 0 auto; }}
+  h1 {{ font-size: 1.4rem; color: #f8fafc; margin-bottom: 0.25rem; letter-spacing: -0.02em; }}
+  h2 {{ font-size: 0.75rem; color: #475569; text-transform: uppercase; letter-spacing: 0.12em; margin: 2rem 0 0.75rem; border-bottom: 1px solid #1e293b; padding-bottom: 0.4rem; }}
+  .meta {{ color: #475569; font-size: 0.78rem; margin-bottom: 2rem; }}
+  .meta span {{ margin-right: 1.5rem; }}
+  .score-block {{ display: flex; align-items: center; gap: 1.5rem; margin: 1rem 0; }}
+  .score {{ font-size: 2.8rem; font-weight: 700; color: {score_color}; letter-spacing: -0.03em; line-height: 1; }}
+  .score sup {{ font-size: 1rem; color: #475569; font-weight: 400; }}
+  .bar-wrap {{ flex: 1; background: #1e293b; border-radius: 4px; height: 6px; }}
+  .bar-fill {{ height: 6px; border-radius: 4px; background: {score_color}; width: {score}%; transition: width 0.3s; }}
+  .counts {{ display: flex; gap: 1.5rem; font-size: 0.82rem; margin-top: 0.6rem; }}
+  .improved {{ color: #22c55e; }} .regressed {{ color: #ef4444; }} .neutral {{ color: #475569; }}
+  .slice-table {{ width: 100%; border-collapse: collapse; }}
+  .slice-table td {{ padding: 0.35rem 0; font-size: 0.82rem; vertical-align: middle; }}
+  .slice-label {{ width: 200px; padding-right: 1rem; }}
+  .slice-bar-cell {{ width: 200px; }}
+  .slice-bar-wrap {{ background: #1e293b; border-radius: 2px; height: 5px; }}
+  .slice-score {{ width: 3rem; text-align: right; padding: 0 0.75rem; font-variant-numeric: tabular-nums; }}
+  .slice-n {{ color: #334155; font-size: 0.75rem; }}
+  .example {{ background: #0d1829; border: 1px solid #1e293b; border-radius: 8px; padding: 1.1rem 1.25rem; margin: 0.75rem 0; }}
+  .example + .example {{ margin-top: 0.5rem; }}
+  .example-slot {{ font-size: 0.68rem; text-transform: uppercase; letter-spacing: 0.1em; color: #334155; margin-bottom: 0.6rem; }}
+  .example-tags {{ font-size: 0.75rem; color: #475569; margin-bottom: 0.6rem; }}
+  .example-tags .tag-key {{ color: #334155; }}
+  .example-tags .tag-val {{ color: #64748b; }}
+  .example-row {{ display: grid; grid-template-columns: 3rem 1fr; gap: 0.5rem; font-size: 0.82rem; margin: 0.25rem 0; align-items: baseline; }}
+  .row-key {{ color: #334155; font-size: 0.75rem; }}
+  .row-v1 {{ color: #64748b; }}
+  .row-why {{ color: #475569; font-style: italic; font-size: 0.8rem; }}
+  .divider {{ border: none; border-top: 1px solid #1e293b; margin: 2rem 0; }}
+  .verdict-block {{ border-radius: 8px; padding: 1.25rem 1.5rem; margin-top: 1rem; border-left: 3px solid {verdict_color}; background: #0d1829; }}
+  .verdict-label {{ font-size: 1.1rem; font-weight: 700; color: {verdict_color}; letter-spacing: 0.04em; }}
+  .verdict-rec {{ color: #64748b; font-size: 0.82rem; margin-top: 0.5rem; line-height: 1.6; }}
+  .cluster-wrap {{ display: flex; flex-wrap: wrap; gap: 0.4rem; margin-top: 0.25rem; }}
+  .cluster {{ background: #1e293b; border: 1px solid #2d3f55; border-radius: 4px; padding: 0.2rem 0.65rem; font-size: 0.78rem; color: #64748b; }}
+  .prompt-box {{ background: #0d1829; border: 1px solid #1e293b; border-radius: 6px; padding: 0.75rem 1rem; font-size: 0.8rem; color: #94a3b8; white-space: pre-wrap; word-break: break-word; max-height: 100px; overflow: auto; }}
+  .prompt-label {{ font-size: 0.72rem; color: #334155; text-transform: uppercase; letter-spacing: 0.08em; margin: 0.75rem 0 0.3rem; }}
+</style>
+</head>
+<body>
+
+<h1>diffprompt</h1>
+<div class="meta">
+  <span>model: {_esc(report.model)}</span>
+  <span>judge: {_esc(report.judge)}</span>
+  <span>tests: {len(report.test_cases)}</span>
+  <span>diversity: {report.diversity_score:.2f}</span>
+</div>
+
+<h2>Prompts</h2>
+<div class="prompt-label">v1 — baseline</div>
+<div class="prompt-box">{_esc(report.prompt_v1)}</div>
+<div class="prompt-label">v2 — candidate</div>
+<div class="prompt-box">{_esc(report.prompt_v2)}</div>
+
+<h2>Summary</h2>
+<div class="score-block">
+  <div class="score">{score}<sup>/100</sup></div>
+  <div style="flex:1">
+    <div class="bar-wrap"><div class="bar-fill"></div></div>
+    <div class="counts">
+      <span class="improved">▲ {report.n_improved} improved</span>
+      <span class="regressed">▼ {report.n_regressed} regressed</span>
+      <span class="neutral">→ {report.n_neutral} neutral</span>
+    </div>
+  </div>
+</div>
+
+{slices_html}
+{examples_html}
+{clusters_html}
+
+<h2>Verdict</h2>
+<div class="verdict-block">
+  <div class="verdict-label">{verdict_label}</div>
+  <div class="verdict-rec">{_esc(report.recommendation)}</div>
+</div>
+
+</body>
+</html>"""
+
+
+def _render_slices(report: DiffReport) -> str:
+    slices = [s for s in report.slices if s.depth == 1]
+    if not slices:
+        return ""
+
+    # Normalize bar widths to the actual min/max range so differences are visible
+    scores = [s.mean_similarity for s in slices]
+    lo, hi = min(scores), max(scores)
+    span   = (hi - lo) or 0.01
+
+    def bar_pct(sim: float) -> int:
+        # 15–95% range so bars are always visible and differences readable
+        return int(15 + ((sim - lo) / span) * 80)
+
+    wins  = [s for s in slices if s.verdict == Verdict.IMPROVEMENT]
+    fails = [s for s in slices if s.verdict == Verdict.REGRESSION]
+
+    rows = []
+    if wins:
+        rows.append('<tr><td colspan="4" style="padding-top:0.5rem;font-size:0.75rem;color:#22c55e;padding-bottom:0.2rem">v2 performs well when…</td></tr>')
+        for s in wins[:4]:
+            rows.append(_slice_row(s, "#22c55e", bar_pct(s.mean_similarity)))
+    if fails:
+        rows.append('<tr><td colspan="4" style="padding-top:0.75rem;font-size:0.75rem;color:#ef4444;padding-bottom:0.2rem">v2 struggles when…</td></tr>')
+        for s in fails[:6]:
+            rows.append(_slice_row(s, "#ef4444", bar_pct(s.mean_similarity)))
+
+    return f"<h2>Behavioral Profile</h2><table class='slice-table'>{''.join(rows)}</table>"
+
+
+def _slice_row(s, color: str, bar_pct: int) -> str:
+    warn = ""
+    if s.n < 3:
+        warn = '<span style="color:#f59e0b;font-size:0.72rem"> ⚠ low n</span>'
+    elif s.typical_ratio == 0:
+        warn = '<span style="color:#f59e0b;font-size:0.72rem"> ⚠ no typical tests</span>'
+    return f"""<tr>
+      <td class="slice-label" style="color:{color}">{_esc(s.label)}</td>
+      <td class="slice-bar-cell">
+        <div class="slice-bar-wrap">
+          <div style="height:5px;border-radius:2px;background:{color};width:{bar_pct}%"></div>
+        </div>
+      </td>
+      <td class="slice-score" style="color:{color}">{s.mean_similarity:.2f}</td>
+      <td class="slice-n">{s.n} tests{warn}</td>
+    </tr>"""
+
+
+def _render_examples(report: DiffReport) -> str:
+    if not report.key_examples:
+        return ""
+
+    slot_label = {
+        "most_important":   "Most Important",
+        "best_improvement": "Best Improvement",
+        "most_surprising":  "Most Surprising",
+    }
+
+    cards = []
+    for ex in report.key_examples:
+        d     = ex.diff
+        color = _VERDICT_COLOR[d.verdict]
+        label = slot_label.get(ex.slot, ex.slot.replace("_", " ").title())
+        tags  = "  ".join(
+            f'<span class="tag-key">{_esc(k)}:</span><span class="tag-val">{_esc(v)}</span>'
+            for k, v in d.test_case.tags.items()
+        )
+        div_color = _VERDICT_COLOR[d.verdict]
+        cards.append(f"""
+<div class="example" style="border-left: 3px solid {div_color}">
+  <div class="example-slot">{label}</div>
+  <div class="example-tags">{tags} &nbsp;·&nbsp; divergence <span style="color:{div_color}">{d.divergence:.2f}</span></div>
+  <div class="example-row"><span class="row-key">input</span><span>{_esc(d.test_case.input[:140])}</span></div>
+  <div class="example-row"><span class="row-key">v1</span><span class="row-v1">{_esc(d.v1_output[:220])}</span></div>
+  <div class="example-row"><span class="row-key">v2</span><span style="color:{div_color}">{_esc(d.v2_output[:220])}</span></div>
+  <div class="example-row"><span class="row-key">why</span><span class="row-why">{_esc(ex.why_it_matters)}</span></div>
+</div>""")
+
+    return f"<h2>Key Examples</h2>{''.join(cards)}"
+
+
+def _render_clusters(report: DiffReport) -> str:
+    if not report.clusters:
+        return ""
+
+    # Deduplicate: merge clusters with the same name, summing their counts
+    merged: dict[str, int] = defaultdict(int)
+    for c in report.clusters:
+        merged[c.name] += c.n
+    sorted_clusters = sorted(merged.items(), key=lambda x: x[1], reverse=True)
+
+    tags = "".join(
+        f'<span class="cluster">{_esc(name)} <span style="color:#475569">({n})</span></span>'
+        for name, n in sorted_clusters
+    )
+    return f"<h2>Failure Mode Clusters</h2><div class='cluster-wrap'>{tags}</div>"
+
+
+def _esc(text: str) -> str:
+    return (
+        str(text)
+        .replace("&", "&amp;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+        .lstrip("\ufeff")  # strip BOM if file was read with UTF-8-BOM
+    )

diffprompt/output/terminal.py

@@ -0,0 +1,154 @@
+"""
+Rich terminal output renderer.
+Clean, minimal, fits in one screen.
+"""
+from __future__ import annotations
+from rich.console import Console
+from rich.text import Text
+from rich.rule import Rule
+from diffprompt.models import DiffReport, Verdict, SliceResult
+
+console = Console(highlight=False)
+
+VERDICT_COLOR = {
+    Verdict.IMPROVEMENT: "green",
+    Verdict.REGRESSION:  "red",
+    Verdict.NEUTRAL:     "dim",
+}
+VERDICT_ICON = {
+    Verdict.IMPROVEMENT: "✓",
+    Verdict.REGRESSION:  "✗",
+    Verdict.NEUTRAL:     "→",
+}
+VERDICT_LABEL = {
+    Verdict.IMPROVEMENT: "SHIP IT",
+    Verdict.NEUTRAL:     "CONDITIONAL",
+    Verdict.REGRESSION:  "DO NOT SHIP",
+}
+
+
+def render(report: DiffReport) -> None:
+    console.print()
+    _header(report)
+    _summary(report)
+    _behavioral_profile(report)
+    _key_examples(report)
+    _verdict(report)
+    console.print()
+
+
+def _header(report: DiffReport) -> None:
+    console.print(
+        f"[bold white]diffprompt[/bold white]  [dim]v0.1.0[/dim]  "
+        f"[dim]model:[/dim] [dim]{report.model}[/dim]  "
+        f"[dim]judge:[/dim] [dim]{report.judge}[/dim]  "
+        f"[dim]tests:[/dim] [dim]{len(report.test_cases)}[/dim]"
+    )
+    console.print()
+
+
+def _summary(report: DiffReport) -> None:
+    score  = report.regression_score
+    filled = int(score / 5)
+    bar    = "█" * filled + "░" * (20 - filled)
+    color  = "green" if score >= 80 else "yellow" if score >= 60 else "red"
+
+    console.print(f"[bold]━━ SUMMARY[/bold]")
+    console.print(
+        f"  [{color}]{score}/100[/{color}]  [{color}]{bar}[/{color}]  "
+        f"[green]{report.n_improved} improved[/green]  "
+        f"[red]{report.n_regressed} regressed[/red]  "
+        f"[dim]{report.n_neutral} neutral[/dim]"
+    )
+    console.print()
+
+
+def _behavioral_profile(report: DiffReport) -> None:
+    if not report.slices:
+        return
+
+    depth1 = [s for s in report.slices if s.depth == 1]
+    wins   = [s for s in depth1 if s.verdict == Verdict.IMPROVEMENT]
+    fails  = [s for s in depth1 if s.verdict == Verdict.REGRESSION]
+
+    if not wins and not fails:
+        return
+
+    console.print("[bold]━━ BEHAVIORAL PROFILE[/bold]")
+
+    if wins:
+        console.print("  [green]v2 performs well when...[/green]")
+        for s in wins[:3]:
+            _slice_row(s, "green")
+
+    if fails:
+        console.print("  [red]v2 struggles when...[/red]")
+        for s in fails[:3]:
+            _slice_row(s, "red")
+
+    console.print()
+
+
+def _slice_row(s: SliceResult, color: str) -> None:
+    icon = "✓" if color == "green" else "✗"
+    warn = _warn(s)
+    console.print(
+        f"  [{color}]{icon}[/{color}] [bold]{s.label:<38}[/bold] "
+        f"[dim]score [/dim][{color}]{s.mean_similarity:.2f}[/{color}]"
+        f"[dim]  {s.n} tests[/dim]"
+        + (f"  {warn}" if warn else "")
+    )
+
+
+def _warn(s: SliceResult) -> str:
+    if s.confidence >= 0.7:
+        return ""
+    if s.n < 3:
+        return "[yellow]⚠ low n[/yellow]"
+    if s.typical_ratio == 0:
+        return "[yellow]⚠ no typical tests[/yellow]"
+    if s.variance > 0.15:
+        return "[yellow]⚠ high variance[/yellow]"
+    return ""
+
+
+def _key_examples(report: DiffReport) -> None:
+    if not report.key_examples:
+        return
+
+    console.print("[bold]━━ KEY EXAMPLES[/bold]")
+
+    slot_labels = {
+        "most_important":   "MOST IMPORTANT",
+        "best_improvement": "BEST IMPROVEMENT",
+        "most_surprising":  "MOST SURPRISING",
+    }
+
+    for ex in report.key_examples:
+        d     = ex.diff
+        label = slot_labels.get(ex.slot, ex.slot.upper())
+        color = VERDICT_COLOR[d.verdict]
+        tags  = "  ".join(f"[dim]{k}:[/dim][cyan]{v}[/cyan]" for k, v in d.test_case.tags.items())
+
+        console.print(f"\n  [bold]{label}[/bold]  {tags}  [dim]divergence[/dim] [{color}]{d.divergence:.2f}[/{color}]")
+        console.print(f"  [dim]input[/dim]  {_clip(d.test_case.input, 90)}")
+        console.print(f"  [dim]v1   [/dim]  [dim]{_clip(d.v1_output, 100)}[/dim]")
+        console.print(f"  [dim]v2   [/dim]  [{color}]{_clip(d.v2_output, 100)}[/{color}]")
+        console.print(f"  [dim]why  [/dim]  [italic dim]{_clip(ex.why_it_matters, 160)}[/italic dim]")
+
+    console.print()
+
+
+def _verdict(report: DiffReport) -> None:
+    color = VERDICT_COLOR[report.verdict]
+    icon  = VERDICT_ICON[report.verdict]
+    label = VERDICT_LABEL[report.verdict]
+
+    console.print("[bold]━━ VERDICT[/bold]")
+    console.print(f"  [{color}]{icon} {label}[/{color}]")
+    console.print(f"  [dim]{report.recommendation}[/dim]")
+
+
+def _clip(text: str, n: int) -> str:
+    text = text.replace("\n", " ").strip()
+    return text[:n] + "…" if len(text) > n else text