diffprompt 0.1.0__py3-none-any.whl

diffprompt/__init__.py

@@ -0,0 +1,2 @@
+"""diffprompt — git diff for your prompt's behavior"""
+__version__ = "0.1.0"

diffprompt/cli.py

@@ -0,0 +1,246 @@
+"""
+diffprompt CLI — uses Click directly (no Typer) for cross-version compatibility.
+"""
+from __future__ import annotations
+import asyncio
+import json
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+console = Console()
+
+
+def _load_prompt(value: str) -> str:
+    p = Path(value)
+    if p.exists() and p.is_file():
+        return p.read_text(encoding="utf-8").strip()
+    return value.strip()
+
+
+@click.group()
+def app():
+    """git diff for your prompt's behavior."""
+    pass
+
+
+@app.command()
+@click.argument("prompt_v1")
+@click.argument("prompt_v2")
+@click.option("--auto-generate", is_flag=True, default=False, help="Auto-generate test cases")
+@click.option("--n", default=40, show_default=True, help="Number of test cases")
+@click.option("--test-file", default=None, help="Path to .jsonl test file")
+@click.option("--model", default="groq/llama-3.3-70b-versatile", show_default=True)
+@click.option("--judge", default="local/qwen2.5:7b", show_default=True)
+@click.option("--local-only", is_flag=True, default=False, help="Never call any external API")
+@click.option("--no-judge", is_flag=True, default=False, help="Skip judge, similarity only")
+@click.option("--output", "output_format", default="terminal",
+              type=click.Choice(["terminal", "json", "html"]), show_default=True)
+@click.option("--save", default=None, help="Save report to this file path")
+@click.option("--top-n", default=3, show_default=True)
+@click.option("--quiet", is_flag=True, default=False, help="Score + verdict only")
+@click.option("--verbose", is_flag=True, default=False)
+@click.option("--ci", is_flag=True, default=False, help="Exit 1 if score below threshold")
+@click.option("--threshold", default=75, show_default=True)
+def diff(prompt_v1, prompt_v2, auto_generate, n, test_file, model, judge,
+         local_only, no_judge, output_format, save, top_n, quiet, verbose, ci, threshold):
+    """Diff the behavioral impact of a prompt change.
+
+    \b
+    PROMPT_V1 and PROMPT_V2 can be file paths or inline strings.
+
+    \b
+    Examples:
+      diffprompt diff v1.txt v2.txt --auto-generate
+      diffprompt diff v1.txt v2.txt --auto-generate --n 20
+      diffprompt diff v1.txt v2.txt --test-file inputs.jsonl
+      diffprompt diff v1.txt v2.txt --auto-generate --ci --threshold 75
+    """
+    asyncio.run(_run_diff(
+        prompt_v1=_load_prompt(prompt_v1),
+        prompt_v2=_load_prompt(prompt_v2),
+        auto_generate=auto_generate, n=n, test_file=test_file,
+        model=model, judge=judge, local_only=local_only, no_judge=no_judge,
+        output_format=output_format, save=save, top_n=top_n,
+        quiet=quiet, verbose=verbose, ci=ci, threshold=threshold,
+    ))
+
+
+async def _run_diff(**kwargs):
+    from diffprompt.core.ontology   import Ontology
+    from diffprompt.core.generator  import generate_test_cases, diversity_score
+    from diffprompt.core.runner     import run_both
+    from diffprompt.core.embedder   import batch_similarity
+    from diffprompt.core.judge      import judge_single
+    from diffprompt.core.clusterer  import cluster_diffs
+    from diffprompt.core.slicer     import compute_slices
+    from diffprompt.core.scorer     import regression_score, select_key_examples
+    from diffprompt.models          import DiffResult, DiffReport, Verdict
+    from diffprompt.output.terminal import render
+
+    prompt_v1  = kwargs["prompt_v1"]
+    prompt_v2  = kwargs["prompt_v2"]
+    local_only = kwargs["local_only"]
+
+    with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), transient=True) as p:
+        task = p.add_task("Starting...", total=None)
+
+        p.update(task, description="Inferring ontology...")
+        ontology = Ontology()
+        await ontology.infer(prompt_v1, local_only=local_only)
+        await ontology.build_anchors(prompt_v1, local_only=local_only)
+        p.update(task, description=f"[green]✓[/green] Ontology: {list(ontology.dimensions.keys())}")
+
+        p.update(task, description="Generating test cases...")
+        test_cases = []
+        if kwargs["test_file"]:
+            test_cases = _load_test_file(kwargs["test_file"])
+        elif kwargs["auto_generate"]:
+            test_cases = await generate_test_cases(
+                prompt_v1, n=kwargs["n"], ontology=ontology, local_only=local_only,
+            )
+        else:
+            console.print("[red]✗ Use --auto-generate or --test-file[/red]")
+            raise SystemExit(1)
+
+        if not test_cases:
+            console.print("[red]✗ No test cases generated[/red]")
+            raise SystemExit(1)
+
+        for tc in test_cases:
+            tc.tags = ontology.tag(tc.input)
+
+        div_score = diversity_score(test_cases)
+        p.update(task, description=f"[green]✓[/green] {len(test_cases)} test cases  diversity={div_score:.2f}")
+
+        p.update(task, description="Running both prompts...")
+        v1_results, v2_results = await run_both(test_cases, prompt_v1, prompt_v2, local_only=local_only)
+        p.update(task, description=f"[green]✓[/green] {len(test_cases) * 2} completions done")
+
+        p.update(task, description="Computing semantic diff...")
+        pairs = [(v1_results[tc.id].output, v2_results[tc.id].output) for tc in test_cases]
+        similarities = batch_similarity(pairs)
+
+        diffs = []
+        for i, tc in enumerate(test_cases):
+            sim    = similarities[i]
+            v1_out = v1_results[tc.id].output
+            v2_out = v2_results[tc.id].output
+            if kwargs["no_judge"]:
+                verdict, reason, confidence = Verdict.NEUTRAL, "judge skipped", 1.0
+            else:
+                verdict, reason, confidence = await judge_single(tc, v1_out, v2_out, sim, local_only=local_only)
+            diffs.append(DiffResult(
+                test_case=tc, v1_output=v1_out, v2_output=v2_out,
+                similarity=sim, divergence=1 - sim,
+                verdict=verdict, reason=reason, judge_confidence=confidence,
+            ))
+        p.update(task, description="[green]✓[/green] Diff complete")
+
+        p.update(task, description="Clustering and slicing...")
+        clusters, unclustered = cluster_diffs(diffs)
+        slices       = compute_slices(diffs)
+        score        = regression_score(diffs)
+        key_examples = await select_key_examples(
+            sorted(diffs, key=lambda d: d.divergence, reverse=True)[:20],
+            local_only=local_only,
+        )
+        p.update(task, description="[green]✓[/green] Analysis complete")
+
+    n_improved  = sum(1 for d in diffs if d.verdict == Verdict.IMPROVEMENT)
+    n_regressed = sum(1 for d in diffs if d.verdict == Verdict.REGRESSION)
+    n_neutral   = sum(1 for d in diffs if d.verdict == Verdict.NEUTRAL)
+
+    overall_verdict = _compute_verdict(n_improved, n_regressed, n_neutral)
+    recommendation  = _generate_recommendation(slices, clusters, overall_verdict)
+
+    report = DiffReport(
+        prompt_v1=prompt_v1, prompt_v2=prompt_v2, model=kwargs["model"], judge=kwargs["judge"],
+        test_cases=test_cases, diversity_score=div_score, diffs=diffs,
+        slices=slices, clusters=clusters, unclustered=unclustered,
+        key_examples=key_examples, regression_score=score,
+        n_improved=n_improved, n_regressed=n_regressed, n_neutral=n_neutral,
+        verdict=overall_verdict, recommendation=recommendation,
+    )
+
+    if not kwargs["quiet"]:
+        render(report)
+    else:
+        console.print(f"score: {score}  verdict: {overall_verdict.value}")
+
+    if kwargs["save"]:
+        _save_report(report, kwargs["save"], kwargs["output_format"])
+        console.print(f"[dim]Saved → {kwargs['save']}[/dim]")
+
+    if kwargs["ci"] and score < kwargs["threshold"]:
+        console.print(f"[red]✗ CI: score {score} < threshold {kwargs['threshold']}[/red]")
+        raise SystemExit(1)
+
+
+def _compute_verdict(n_improved, n_regressed, n_neutral):
+    from diffprompt.models import Verdict
+    score_ratio = n_improved / max(1, n_improved + n_regressed + n_neutral)
+    if n_improved > n_regressed and score_ratio >= 0.6:
+        return Verdict.IMPROVEMENT
+    if n_regressed > n_improved:
+        return Verdict.REGRESSION
+    return Verdict.NEUTRAL
+
+
+def _generate_recommendation(slices, clusters, verdict) -> str:
+    from diffprompt.models import Verdict
+    if not slices:
+        return "Not enough data. Try --n 40 or higher."
+    worst = [s for s in slices if s.verdict == Verdict.REGRESSION and s.depth == 1][:2]
+    best  = [s for s in slices if s.verdict == Verdict.IMPROVEMENT and s.depth == 1][:2]
+    parts = []
+    if best:
+        parts.append(f"Safe to ship v2 for {', '.join(s.label for s in best)}.")
+    if worst:
+        parts.append(f"Keep v1 for {', '.join(s.label for s in worst)}.")
+    if clusters:
+        parts.append(f"Primary failure mode: {clusters[0].name} ({clusters[0].n} cases).")
+    return " ".join(parts) if parts else "Review key examples before shipping."
+
+
+def _load_test_file(path: str) -> list:
+    import uuid
+    from diffprompt.models import TestCase, TestCategory
+    cases = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            data = json.loads(line)
+            inp = data.get("input") or data.get("text") or data.get("prompt", "")
+            if inp:
+                cases.append(TestCase(id=str(uuid.uuid4()), input=inp, category=TestCategory.TYPICAL))
+    return cases
+
+
+def _save_report(report, path: str, fmt: str) -> None:
+    if fmt == "json" or path.endswith(".json"):
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(report.model_dump_json(indent=2))
+    elif fmt == "html" or path.endswith(".html"):
+        from diffprompt.output.exporter import render_html
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(render_html(report))
+    else:
+        lines = [
+            "diffprompt report", "=" * 40,
+            f"v1: {report.prompt_v1[:80]}", f"v2: {report.prompt_v2[:80]}",
+            f"score: {report.regression_score}/100",
+            f"verdict: {report.verdict.value.upper()}",
+            f"improved: {report.n_improved}  regressed: {report.n_regressed}  neutral: {report.n_neutral}",
+            "", f"recommendation: {report.recommendation}",
+        ]
+        with open(path, "w", encoding="utf-8") as f:
+            f.write("\n".join(lines))
+
+
+if __name__ == "__main__":
+    app()

diffprompt/core/clusterer.py

@@ -0,0 +1,130 @@
+"""
+Clusters diff results by embedding judge reasons.
+Uses HDBSCAN + UMAP. Returns named failure modes.
+"""
+from __future__ import annotations
+import numpy as np
+from collections import defaultdict
+from diffprompt.models import DiffResult, Cluster, Verdict
+from diffprompt.core.embedder import embed
+
+# Below this count, clustering is meaningless — return everything as unclustered
+_MIN_CLUSTER_INPUT = 10
+
+
+def cluster_diffs(diffs: list[DiffResult]) -> tuple[list[Cluster], list[DiffResult]]:
+    """
+    Cluster diffs by their judge reasons.
+    Returns (clusters, unclustered) where unclustered = HDBSCAN noise (label -1).
+    """
+    try:
+        import hdbscan
+        import umap
+    except ImportError:
+        raise ImportError("Run: pip install hdbscan umap-learn")
+
+    if len(diffs) < _MIN_CLUSTER_INPUT:
+        return [], diffs
+
+    reasons = [d.reason for d in diffs]
+    embs = embed(reasons)
+
+    # UMAP: reduce to low-dim before HDBSCAN (better cluster quality)
+    n_components = min(5, len(diffs) - 2)
+    reducer = umap.UMAP(n_components=n_components, random_state=42, verbose=False)
+    reduced = reducer.fit_transform(embs)
+
+    clusterer = hdbscan.HDBSCAN(min_cluster_size=2, min_samples=1)
+    labels = clusterer.fit_predict(reduced)
+
+    # Compute centrality per point within its cluster
+    centrality_map = _compute_centrality(embs, labels)
+
+    groups: dict[int, list[int]] = defaultdict(list)
+    for i, label in enumerate(labels):
+        groups[label].append(i)
+
+    clusters = []
+    unclustered = []
+
+    for label, indices in groups.items():
+        group_diffs = [diffs[i] for i in indices]
+
+        # Update cluster metadata on each diff
+        for i_local, i_global in enumerate(indices):
+            diffs[i_global].cluster_label = label
+            diffs[i_global].cluster_centrality = centrality_map.get(i_global, 0.0)
+
+        if label == -1:
+            unclustered.extend(group_diffs)
+            continue
+
+        cluster = Cluster(
+            label=label,
+            name=_name_cluster(label, group_diffs),
+            description=_describe_cluster(group_diffs),
+            n=len(group_diffs),
+            mean_similarity=float(np.mean([d.similarity for d in group_diffs])),
+            test_ids=[d.test_case.id for d in group_diffs],
+        )
+        clusters.append(cluster)
+
+    clusters.sort(key=lambda c: c.n, reverse=True)
+    return clusters, unclustered
+
+
+def _compute_centrality(embs: np.ndarray, labels: np.ndarray) -> dict[int, float]:
+    """
+    For each point, compute its mean cosine similarity to others in its cluster.
+    Returns a dict: index → centrality score (0-1).
+    """
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    centrality = {}
+    unique_labels = set(labels)
+
+    for label in unique_labels:
+        if label == -1:
+            continue
+        indices = [i for i, l in enumerate(labels) if l == label]
+        if len(indices) < 2:
+            for i in indices:
+                centrality[i] = 1.0
+            continue
+        cluster_embs = embs[indices]
+        sim_matrix = cosine_similarity(cluster_embs)
+        np.fill_diagonal(sim_matrix, 0)
+        mean_sims = sim_matrix.mean(axis=1)
+        for i_local, i_global in enumerate(indices):
+            centrality[i_global] = float(mean_sims[i_local])
+
+    return centrality
+
+
+def _name_cluster(label: int, diffs: list[DiffResult]) -> str:
+    """Generate a named failure mode from dominant reason keywords."""
+    reasons = " ".join(d.reason.lower() for d in diffs)
+
+    if any(w in reasons for w in ["brief", "short", "concise", "terse", "succinct"]):
+        return "BREVITY_GAIN" if _is_mostly_improvements(diffs) else "BREVITY_LOSS"
+    if any(w in reasons for w in ["context", "nuance", "detail", "omit", "missing", "incomplete"]):
+        return "CONTEXT_LOSS"
+    if any(w in reasons for w in ["refus", "declin", "won't", "cannot", "avoid"]):
+        return "REFUSAL_SHIFT"
+    if any(w in reasons for w in ["tone", "empathy", "warm", "cold", "formal", "harsh"]):
+        return "TONE_SHIFT"
+    if any(w in reasons for w in ["accura", "wrong", "incorrect", "error", "fact", "hallucin"]):
+        return "ACCURACY_CHANGE"
+    if any(w in reasons for w in ["verbos", "long", "padded", "unnecessar", "redundant"]):
+        return "VERBOSITY_GAIN"
+    if any(w in reasons for w in ["format", "structur", "bullet", "list", "markdown"]):
+        return "FORMAT_CHANGE"
+    return f"CLUSTER_{label}"
+
+
+def _describe_cluster(diffs: list[DiffResult]) -> str:
+    return ". ".join(d.reason for d in diffs[:3])[:120]
+
+
+def _is_mostly_improvements(diffs: list[DiffResult]) -> bool:
+    return sum(1 for d in diffs if d.verdict == Verdict.IMPROVEMENT) > len(diffs) / 2

diffprompt/core/embedder.py

@@ -0,0 +1,63 @@
+"""
+Embedding + similarity layer.
+All local, all free. No API calls.
+"""
+from __future__ import annotations
+import logging
+import os
+import warnings
+import numpy as np
+from functools import lru_cache
+
+# Suppress noisy warnings from HuggingFace / sentence-transformers / UMAP
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
+logging.getLogger("transformers").setLevel(logging.ERROR)
+logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
+warnings.filterwarnings("ignore", message=".*n_jobs value.*overridden.*")
+warnings.filterwarnings("ignore", message=".*unauthenticated.*")
+warnings.filterwarnings("ignore", message=".*UNEXPECTED.*")
+warnings.filterwarnings("ignore", category=UserWarning, module="umap")
+
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity as sk_cosine
+
+
+MODEL_NAME = "all-MiniLM-L6-v2"
+
+
+@lru_cache(maxsize=1)
+def get_embedder() -> SentenceTransformer:
+    """Lazy-load embedder. Cached so it only loads once per session."""
+    # Silence the load report printed to stdout by newer sentence-transformers
+    import io, contextlib
+    f = io.StringIO()
+    with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f):
+        model = SentenceTransformer(MODEL_NAME)
+    return model
+
+
+def embed(texts: list[str]) -> np.ndarray:
+    return get_embedder().encode(texts, show_progress_bar=False)
+
+
+def similarity(text_a: str, text_b: str) -> float:
+    """Cosine similarity between two texts. Returns float 0-1."""
+    embs = embed([text_a, text_b])
+    score = sk_cosine([embs[0]], [embs[1]])[0][0]
+    return float(np.clip(score, 0, 1))
+
+
+def batch_similarity(pairs: list[tuple[str, str]]) -> list[float]:
+    """
+    Efficient batch similarity for many pairs.
+    Embeds all texts in one pass instead of N passes.
+    """
+    all_texts = [t for pair in pairs for t in pair]
+    all_embs = embed(all_texts)
+    scores = []
+    for i in range(0, len(all_embs), 2):
+        score = sk_cosine([all_embs[i]], [all_embs[i + 1]])[0][0]
+        scores.append(float(np.clip(score, 0, 1)))
+    return scores

diffprompt/core/generator.py

@@ -0,0 +1,123 @@
+"""
+Test case generation using taxonomy-based prompting.
+Four buckets: typical, boundary, adversarial, format.
+"""
+from __future__ import annotations
+import json
+import re
+import uuid
+from typing import Optional
+
+from diffprompt.models import TestCase, TestCategory
+from diffprompt.models.cascade import call_cascade
+from diffprompt.core.ontology import Ontology
+
+
+TAXONOMY_PROMPTS = {
+    TestCategory.TYPICAL: """Generate {n} realistic inputs a real user would send to this prompt.
+These should reflect actual everyday usage — diverse topics, normal phrasing.
+Prompt: {prompt}
+Return ONLY a JSON array of strings. No explanation.""",
+
+    TestCategory.BOUNDARY: """Generate {n} inputs at the edges of what this prompt handles.
+Too long, too short, tangentially related, slightly out of scope.
+The prompt should handle these but might struggle.
+Prompt: {prompt}
+Return ONLY a JSON array of strings. No explanation.""",
+
+    TestCategory.ADVERSARIAL: """Generate {n} inputs designed to expose inconsistencies or failures.
+Use ambiguous phrasing, contradictory requirements, edge cases, trick questions.
+Make the prompt work hard.
+Prompt: {prompt}
+Return ONLY a JSON array of strings. No explanation.""",
+
+    TestCategory.FORMAT: """Generate {n} inputs with unusual formatting.
+Try: ALL CAPS, no punctuation, mixed languages, extremely long sentences,
+bullet points as input, emojis, very short (1-2 words), JSON-formatted input.
+Prompt: {prompt}
+Return ONLY a JSON array of strings. No explanation.""",
+}
+
+# Distribution: 45% typical, 35% adversarial, 10% boundary, 10% format
+DISTRIBUTION = {
+    TestCategory.TYPICAL:     0.45,
+    TestCategory.ADVERSARIAL: 0.35,
+    TestCategory.BOUNDARY:    0.10,
+    TestCategory.FORMAT:      0.10,
+}
+
+
+async def generate_test_cases(
+    prompt: str,
+    n: int = 40,
+    ontology: Optional[Ontology] = None,
+    local_only: bool = False,
+) -> list[TestCase]:
+    """Generate n test cases across all taxonomy buckets."""
+    test_cases: list[TestCase] = []
+
+    for category, fraction in DISTRIBUTION.items():
+        count = max(2, round(n * fraction))
+
+        inputs = await _generate_bucket(prompt, category, count, local_only)
+
+        for inp in inputs[:count]:
+            test_cases.append(TestCase(
+                id=str(uuid.uuid4()),
+                input=inp,
+                category=category,
+            ))
+
+    # BUG FIX: return was previously inside the for loop,
+    # causing only the first bucket (typical) to ever be generated.
+    return test_cases
+
+
+async def _generate_bucket(
+    prompt: str,
+    category: TestCategory,
+    count: int,
+    local_only: bool,
+) -> list[str]:
+    """Generate one bucket of test cases with up to 2 retries on parse failure."""
+    for attempt in range(2):
+        raw, _ = await call_cascade(
+            TAXONOMY_PROMPTS[category].format(n=count, prompt=prompt),
+            local_only=local_only,
+        )
+        clean = re.sub(r"```(?:json)?|```", "", raw).strip()
+        match = re.search(r'\[.*\]', clean, re.DOTALL)
+        if match:
+            clean = match.group(0)
+        try:
+            parsed = json.loads(clean)
+            if isinstance(parsed, list):
+                return [str(x) for x in parsed if x]
+        except json.JSONDecodeError:
+            if attempt == 1:
+                return []  # give up after 2 attempts
+
+    return []
+
+
+def diversity_score(test_cases: list[TestCase]) -> float:
+    """
+    Compute diversity score for the test suite.
+    1 - mean pairwise similarity. Higher = more diverse.
+    Uses the shared embedder from diffprompt.core.embedder to avoid
+    loading a second model instance.
+    """
+    from diffprompt.core.embedder import embed
+    from sklearn.metrics.pairwise import cosine_similarity
+    import numpy as np
+
+    if len(test_cases) < 2:
+        return 1.0
+
+    inputs = [tc.input for tc in test_cases]
+    embs = embed(inputs)
+    sim_matrix = cosine_similarity(embs)
+    np.fill_diagonal(sim_matrix, 0)
+    n = len(inputs)
+    mean_sim = sim_matrix.sum() / (n * (n - 1))
+    return float(1 - mean_sim)

diffprompt/core/judge.py

@@ -0,0 +1,94 @@
+"""
+LLM-as-judge layer.
+Determines verdict (improvement/regression/neutral) + reason per diff.
+Escalates to larger Groq model when confidence is low.
+"""
+from __future__ import annotations
+import json
+import re
+from diffprompt.models import Verdict, DiffResult, TestCase
+from diffprompt.models.cascade import call_cascade, call_groq_only
+
+
+JUDGE_PROMPT = """You are evaluating whether a prompt change improved or worsened an output.
+
+Input: {input}
+
+Output V1:
+{v1}
+
+Output V2:
+{v2}
+
+Which output better serves this input? Consider: accuracy, completeness, tone, and appropriateness.
+Be specific. One sentence reason.
+
+Respond ONLY with valid JSON (no markdown fences):
+{{"verdict": "improvement|regression|neutral", "reason": "one sentence explaining the key difference", "confidence": 0.0}}
+
+confidence should reflect how clear-cut the verdict is (1.0 = obvious, 0.5 = ambiguous)."""
+
+CONFIDENCE_THRESHOLD = 0.65  # escalate to larger model below this
+
+
+async def judge_single(
+    test_case: TestCase,
+    v1_output: str,
+    v2_output: str,
+    similarity: float,
+    local_only: bool = False,
+) -> tuple[Verdict, str, float]:
+    """
+    Returns (verdict, reason, confidence).
+    Auto-escalates to Groq 70B if confidence is low.
+    """
+    # Fast path: very high similarity → neutral, skip judge entirely
+    if similarity > 0.95:
+        return Verdict.NEUTRAL, "outputs are semantically identical", 1.0
+
+    prompt = JUDGE_PROMPT.format(
+        input=test_case.input,
+        v1=v1_output[:800],   # truncate to avoid token waste
+        v2=v2_output[:800],
+    )
+
+    raw, _ = await call_cascade(prompt, local_only=local_only)
+    verdict, reason, confidence = _parse_judge_response(raw)
+
+    # Escalate if confidence is low and we're not local-only.
+    # BUG FIX: previously called call_cascade() again which would re-try
+    # Ollama first — defeating the purpose of escalation. Now calls
+    # call_groq_only() directly to guarantee a larger model.
+    if confidence < CONFIDENCE_THRESHOLD and not local_only:
+        escalated = await call_groq_only(
+            prompt,
+            groq_model="llama-3.3-70b-versatile",
+        )
+        if escalated:
+            verdict, reason, confidence = _parse_judge_response(escalated)
+
+    return verdict, reason, confidence
+
+
+def _parse_judge_response(raw: str) -> tuple[Verdict, str, float]:
+    """Parse judge JSON response with fallback."""
+    try:
+        clean = re.sub(r"```(?:json)?|```", "", raw).strip()
+        # Handle responses where the model adds text before/after JSON
+        match = re.search(r'\{.*\}', clean, re.DOTALL)
+        if match:
+            clean = match.group(0)
+        data = json.loads(clean)
+        verdict_str = data.get("verdict", "neutral").lower().strip()
+        # Normalize common variants
+        if verdict_str in ("improve", "improved", "better"):
+            verdict_str = "improvement"
+        elif verdict_str in ("regress", "regressed", "worse"):
+            verdict_str = "regression"
+        verdict = Verdict(verdict_str) if verdict_str in Verdict._value2member_map_ else Verdict.NEUTRAL
+        reason = str(data.get("reason", "no reason provided"))[:300]
+        confidence = float(data.get("confidence", 0.5))
+        confidence = max(0.0, min(1.0, confidence))
+        return verdict, reason, confidence
+    except Exception:
+        return Verdict.NEUTRAL, "judge parse error — raw response was not valid JSON", 0.0