diffprompt 0.1.0__tar.gz → 0.2.0__tar.gz

diffprompt-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest tests/ -v

{diffprompt-0.1.0 → diffprompt-0.2.0}/.gitignore

@@ -1,10 +1,12 @@
-__pycache__/
-*.py[cod]
-*.egg-info/
-dist/
-build/
-.env
-.venv/
-venv/
-diffprompt_report.*
-diffprompt.ontology.json
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.env
+.venv/
+venv/
+diffprompt_report.*
+diffprompt.ontology.json
+.diffprompt_cache/
+.superpowers/

{diffprompt-0.1.0 → diffprompt-0.2.0}/API_REFERENCE.md

@@ -299,7 +299,7 @@ Generates `n` test cases distributed across the four taxonomy buckets (45% typic
 #### diversity_score
 
 ```python
-def diversity_score(test_cases: list[TestCase], embedder: SentenceTransformer) -> float
+def diversity_score(test_cases: list[TestCase]) -> float
 ```
 
 Computes how diverse the test suite is. Returns `1 - mean_pairwise_similarity`. Higher is more diverse. A score below 0.4 means many inputs are semantically redundant.
@@ -485,6 +485,7 @@ where `surprise = divergence * (1 - input_length / 50)`. Short inputs that chang
 ```python
 async def select_key_examples(
     diffs: list[DiffResult],
+    top_n: int = 3,
     local_only: bool = False,
 ) -> list[KeyExample]
 ```

{diffprompt-0.1.0 → diffprompt-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: diffprompt
-Version: 0.1.0
+Version: 0.2.0
 Summary: git diff for your prompt's behavior
 Project-URL: Homepage, https://github.com/RudraDudhat2509/diffprompt
 Project-URL: Repository, https://github.com/RudraDudhat2509/diffprompt
@@ -30,6 +30,8 @@ Description-Content-Type: text/markdown
 
 # diffprompt
 
+[![CI](https://github.com/RudraDudhat2509/diffprompt/actions/workflows/ci.yml/badge.svg)](https://github.com/RudraDudhat2509/diffprompt/actions/workflows/ci.yml)
+
 > git diff for your prompt's behavior
 
 ```bash
@@ -161,6 +163,12 @@ Different job. Different tool.
 
 Override any layer with `--model` and `--judge`.
 
+Groq calls are paced to stay under the free-tier rate limit. Tune it with `DIFFPROMPT_GROQ_RPM` (default 30; set `0` to disable):
+
+```bash
+export DIFFPROMPT_GROQ_RPM=60   # raise on paid tiers
+```
+
 ---
 
 ## CLI reference

{diffprompt-0.1.0 → diffprompt-0.2.0}/README.md

@@ -1,5 +1,7 @@
 # diffprompt
 
+[![CI](https://github.com/RudraDudhat2509/diffprompt/actions/workflows/ci.yml/badge.svg)](https://github.com/RudraDudhat2509/diffprompt/actions/workflows/ci.yml)
+
 > git diff for your prompt's behavior
 
 ```bash
@@ -131,6 +133,12 @@ Different job. Different tool.
 
 Override any layer with `--model` and `--judge`.
 
+Groq calls are paced to stay under the free-tier rate limit. Tune it with `DIFFPROMPT_GROQ_RPM` (default 30; set `0` to disable):
+
+```bash
+export DIFFPROMPT_GROQ_RPM=60   # raise on paid tiers
+```
+
 ---
 
 ## CLI reference

{diffprompt-0.1.0 → diffprompt-0.2.0}/diffprompt/__init__.py

@@ -1,2 +1,2 @@
-"""diffprompt — git diff for your prompt's behavior"""
-__version__ = "0.1.0"
+"""diffprompt — git diff for your prompt's behavior"""
+__version__ = "0.2.0"

{diffprompt-0.1.0 → diffprompt-0.2.0}/diffprompt/cli.py

@@ -73,7 +73,7 @@ async def _run_diff(**kwargs):
     from diffprompt.core.generator  import generate_test_cases, diversity_score
     from diffprompt.core.runner     import run_both
     from diffprompt.core.embedder   import batch_similarity
-    from diffprompt.core.judge      import judge_single
+    from diffprompt.core.judge      import judge_all
     from diffprompt.core.clusterer  import cluster_diffs
     from diffprompt.core.slicer     import compute_slices
     from diffprompt.core.scorer     import regression_score, select_key_examples
@@ -116,26 +116,32 @@ async def _run_diff(**kwargs):
         p.update(task, description=f"[green]✓[/green] {len(test_cases)} test cases  diversity={div_score:.2f}")
 
         p.update(task, description="Running both prompts...")
-        v1_results, v2_results = await run_both(test_cases, prompt_v1, prompt_v2, local_only=local_only)
+        v1_results, v2_results = await run_both(
+            test_cases, prompt_v1, prompt_v2, model=kwargs["model"], local_only=local_only,
+        )
         p.update(task, description=f"[green]✓[/green] {len(test_cases) * 2} completions done")
 
         p.update(task, description="Computing semantic diff...")
-        pairs = [(v1_results[tc.id].output, v2_results[tc.id].output) for tc in test_cases]
-        similarities = batch_similarity(pairs)
+        v1_outputs = [v1_results[tc.id].output for tc in test_cases]
+        v2_outputs = [v2_results[tc.id].output for tc in test_cases]
+        similarities = batch_similarity(list(zip(v1_outputs, v2_outputs)))
+
+        if kwargs["no_judge"]:
+            judgements = [(Verdict.NEUTRAL, "judge skipped", 1.0) for _ in test_cases]
+        else:
+            judgements = await judge_all(
+                test_cases, v1_outputs, v2_outputs, similarities, local_only=local_only,
+            )
 
         diffs = []
         for i, tc in enumerate(test_cases):
-            sim    = similarities[i]
-            v1_out = v1_results[tc.id].output
-            v2_out = v2_results[tc.id].output
-            if kwargs["no_judge"]:
-                verdict, reason, confidence = Verdict.NEUTRAL, "judge skipped", 1.0
-            else:
-                verdict, reason, confidence = await judge_single(tc, v1_out, v2_out, sim, local_only=local_only)
+            verdict, reason, confidence = judgements[i]
             diffs.append(DiffResult(
-                test_case=tc, v1_output=v1_out, v2_output=v2_out,
-                similarity=sim, divergence=1 - sim,
+                test_case=tc, v1_output=v1_outputs[i], v2_output=v2_outputs[i],
+                similarity=similarities[i], divergence=1 - similarities[i],
                 verdict=verdict, reason=reason, judge_confidence=confidence,
+                v1_latency_ms=v1_results[tc.id].latency_ms,
+                v2_latency_ms=v2_results[tc.id].latency_ms,
             ))
         p.update(task, description="[green]✓[/green] Diff complete")
 
@@ -145,6 +151,7 @@ async def _run_diff(**kwargs):
         score        = regression_score(diffs)
         key_examples = await select_key_examples(
             sorted(diffs, key=lambda d: d.divergence, reverse=True)[:20],
+            top_n=kwargs["top_n"],
             local_only=local_only,
         )
         p.update(task, description="[green]✓[/green] Analysis complete")

{diffprompt-0.1.0 → diffprompt-0.2.0}/diffprompt/core/clusterer.py

@@ -1,130 +1,130 @@
-"""
-Clusters diff results by embedding judge reasons.
-Uses HDBSCAN + UMAP. Returns named failure modes.
-"""
-from __future__ import annotations
-import numpy as np
-from collections import defaultdict
-from diffprompt.models import DiffResult, Cluster, Verdict
-from diffprompt.core.embedder import embed
-
-# Below this count, clustering is meaningless — return everything as unclustered
-_MIN_CLUSTER_INPUT = 10
-
-
-def cluster_diffs(diffs: list[DiffResult]) -> tuple[list[Cluster], list[DiffResult]]:
-    """
-    Cluster diffs by their judge reasons.
-    Returns (clusters, unclustered) where unclustered = HDBSCAN noise (label -1).
-    """
-    try:
-        import hdbscan
-        import umap
-    except ImportError:
-        raise ImportError("Run: pip install hdbscan umap-learn")
-
-    if len(diffs) < _MIN_CLUSTER_INPUT:
-        return [], diffs
-
-    reasons = [d.reason for d in diffs]
-    embs = embed(reasons)
-
-    # UMAP: reduce to low-dim before HDBSCAN (better cluster quality)
-    n_components = min(5, len(diffs) - 2)
-    reducer = umap.UMAP(n_components=n_components, random_state=42, verbose=False)
-    reduced = reducer.fit_transform(embs)
-
-    clusterer = hdbscan.HDBSCAN(min_cluster_size=2, min_samples=1)
-    labels = clusterer.fit_predict(reduced)
-
-    # Compute centrality per point within its cluster
-    centrality_map = _compute_centrality(embs, labels)
-
-    groups: dict[int, list[int]] = defaultdict(list)
-    for i, label in enumerate(labels):
-        groups[label].append(i)
-
-    clusters = []
-    unclustered = []
-
-    for label, indices in groups.items():
-        group_diffs = [diffs[i] for i in indices]
-
-        # Update cluster metadata on each diff
-        for i_local, i_global in enumerate(indices):
-            diffs[i_global].cluster_label = label
-            diffs[i_global].cluster_centrality = centrality_map.get(i_global, 0.0)
-
-        if label == -1:
-            unclustered.extend(group_diffs)
-            continue
-
-        cluster = Cluster(
-            label=label,
-            name=_name_cluster(label, group_diffs),
-            description=_describe_cluster(group_diffs),
-            n=len(group_diffs),
-            mean_similarity=float(np.mean([d.similarity for d in group_diffs])),
-            test_ids=[d.test_case.id for d in group_diffs],
-        )
-        clusters.append(cluster)
-
-    clusters.sort(key=lambda c: c.n, reverse=True)
-    return clusters, unclustered
-
-
-def _compute_centrality(embs: np.ndarray, labels: np.ndarray) -> dict[int, float]:
-    """
-    For each point, compute its mean cosine similarity to others in its cluster.
-    Returns a dict: index → centrality score (0-1).
-    """
-    from sklearn.metrics.pairwise import cosine_similarity
-
-    centrality = {}
-    unique_labels = set(labels)
-
-    for label in unique_labels:
-        if label == -1:
-            continue
-        indices = [i for i, l in enumerate(labels) if l == label]
-        if len(indices) < 2:
-            for i in indices:
-                centrality[i] = 1.0
-            continue
-        cluster_embs = embs[indices]
-        sim_matrix = cosine_similarity(cluster_embs)
-        np.fill_diagonal(sim_matrix, 0)
-        mean_sims = sim_matrix.mean(axis=1)
-        for i_local, i_global in enumerate(indices):
-            centrality[i_global] = float(mean_sims[i_local])
-
-    return centrality
-
-
-def _name_cluster(label: int, diffs: list[DiffResult]) -> str:
-    """Generate a named failure mode from dominant reason keywords."""
-    reasons = " ".join(d.reason.lower() for d in diffs)
-
-    if any(w in reasons for w in ["brief", "short", "concise", "terse", "succinct"]):
-        return "BREVITY_GAIN" if _is_mostly_improvements(diffs) else "BREVITY_LOSS"
-    if any(w in reasons for w in ["context", "nuance", "detail", "omit", "missing", "incomplete"]):
-        return "CONTEXT_LOSS"
-    if any(w in reasons for w in ["refus", "declin", "won't", "cannot", "avoid"]):
-        return "REFUSAL_SHIFT"
-    if any(w in reasons for w in ["tone", "empathy", "warm", "cold", "formal", "harsh"]):
-        return "TONE_SHIFT"
-    if any(w in reasons for w in ["accura", "wrong", "incorrect", "error", "fact", "hallucin"]):
-        return "ACCURACY_CHANGE"
-    if any(w in reasons for w in ["verbos", "long", "padded", "unnecessar", "redundant"]):
-        return "VERBOSITY_GAIN"
-    if any(w in reasons for w in ["format", "structur", "bullet", "list", "markdown"]):
-        return "FORMAT_CHANGE"
-    return f"CLUSTER_{label}"
-
-
-def _describe_cluster(diffs: list[DiffResult]) -> str:
-    return ". ".join(d.reason for d in diffs[:3])[:120]
-
-
-def _is_mostly_improvements(diffs: list[DiffResult]) -> bool:
+"""
+Clusters diff results by embedding judge reasons.
+Uses HDBSCAN + UMAP. Returns named failure modes.
+"""
+from __future__ import annotations
+import numpy as np
+from collections import defaultdict
+from diffprompt.models import DiffResult, Cluster, Verdict
+from diffprompt.core.embedder import embed
+
+# Below this count, clustering is meaningless — return everything as unclustered
+_MIN_CLUSTER_INPUT = 10
+
+
+def cluster_diffs(diffs: list[DiffResult]) -> tuple[list[Cluster], list[DiffResult]]:
+    """
+    Cluster diffs by their judge reasons.
+    Returns (clusters, unclustered) where unclustered = HDBSCAN noise (label -1).
+    """
+    try:
+        import hdbscan
+        import umap
+    except ImportError:
+        raise ImportError("Run: pip install hdbscan umap-learn")
+
+    if len(diffs) < _MIN_CLUSTER_INPUT:
+        return [], diffs
+
+    reasons = [d.reason for d in diffs]
+    embs = embed(reasons)
+
+    # UMAP: reduce to low-dim before HDBSCAN (better cluster quality)
+    n_components = min(5, len(diffs) - 2)
+    reducer = umap.UMAP(n_components=n_components, random_state=42, verbose=False)
+    reduced = reducer.fit_transform(embs)
+
+    clusterer = hdbscan.HDBSCAN(min_cluster_size=2, min_samples=1)
+    labels = clusterer.fit_predict(reduced)
+
+    # Compute centrality per point within its cluster
+    centrality_map = _compute_centrality(embs, labels)
+
+    groups: dict[int, list[int]] = defaultdict(list)
+    for i, label in enumerate(labels):
+        groups[label].append(i)
+
+    clusters = []
+    unclustered = []
+
+    for label, indices in groups.items():
+        group_diffs = [diffs[i] for i in indices]
+
+        # Update cluster metadata on each diff
+        for i_local, i_global in enumerate(indices):
+            diffs[i_global].cluster_label = label
+            diffs[i_global].cluster_centrality = centrality_map.get(i_global, 0.0)
+
+        if label == -1:
+            unclustered.extend(group_diffs)
+            continue
+
+        cluster = Cluster(
+            label=label,
+            name=_name_cluster(label, group_diffs),
+            description=_describe_cluster(group_diffs),
+            n=len(group_diffs),
+            mean_similarity=float(np.mean([d.similarity for d in group_diffs])),
+            test_ids=[d.test_case.id for d in group_diffs],
+        )
+        clusters.append(cluster)
+
+    clusters.sort(key=lambda c: c.n, reverse=True)
+    return clusters, unclustered
+
+
+def _compute_centrality(embs: np.ndarray, labels: np.ndarray) -> dict[int, float]:
+    """
+    For each point, compute its mean cosine similarity to others in its cluster.
+    Returns a dict: index → centrality score (0-1).
+    """
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    centrality = {}
+    unique_labels = set(labels)
+
+    for label in unique_labels:
+        if label == -1:
+            continue
+        indices = [i for i, l in enumerate(labels) if l == label]
+        if len(indices) < 2:
+            for i in indices:
+                centrality[i] = 1.0
+            continue
+        cluster_embs = embs[indices]
+        sim_matrix = cosine_similarity(cluster_embs)
+        np.fill_diagonal(sim_matrix, 0)
+        mean_sims = sim_matrix.mean(axis=1)
+        for i_local, i_global in enumerate(indices):
+            centrality[i_global] = float(mean_sims[i_local])
+
+    return centrality
+
+
+def _name_cluster(label: int, diffs: list[DiffResult]) -> str:
+    """Generate a named failure mode from dominant reason keywords."""
+    reasons = " ".join(d.reason.lower() for d in diffs)
+
+    if any(w in reasons for w in ["brief", "short", "concise", "terse", "succinct"]):
+        return "BREVITY_GAIN" if _is_mostly_improvements(diffs) else "BREVITY_LOSS"
+    if any(w in reasons for w in ["context", "nuance", "detail", "omit", "missing", "incomplete"]):
+        return "CONTEXT_LOSS"
+    if any(w in reasons for w in ["refus", "declin", "won't", "cannot", "avoid"]):
+        return "REFUSAL_SHIFT"
+    if any(w in reasons for w in ["tone", "empathy", "warm", "cold", "formal", "harsh"]):
+        return "TONE_SHIFT"
+    if any(w in reasons for w in ["accura", "wrong", "incorrect", "error", "fact", "hallucin"]):
+        return "ACCURACY_CHANGE"
+    if any(w in reasons for w in ["verbos", "long", "padded", "unnecessar", "redundant"]):
+        return "VERBOSITY_GAIN"
+    if any(w in reasons for w in ["format", "structur", "bullet", "list", "markdown"]):
+        return "FORMAT_CHANGE"
+    return f"CLUSTER_{label}"
+
+
+def _describe_cluster(diffs: list[DiffResult]) -> str:
+    return ". ".join(d.reason for d in diffs[:3])[:120]
+
+
+def _is_mostly_improvements(diffs: list[DiffResult]) -> bool:
     return sum(1 for d in diffs if d.verdict == Verdict.IMPROVEMENT) > len(diffs) / 2

{diffprompt-0.1.0 → diffprompt-0.2.0}/diffprompt/core/embedder.py

@@ -1,63 +1,63 @@
-"""
-Embedding + similarity layer.
-All local, all free. No API calls.
-"""
-from __future__ import annotations
-import logging
-import os
-import warnings
-import numpy as np
-from functools import lru_cache
-
-# Suppress noisy warnings from HuggingFace / sentence-transformers / UMAP
-os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
-os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
-logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
-logging.getLogger("transformers").setLevel(logging.ERROR)
-logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
-warnings.filterwarnings("ignore", message=".*n_jobs value.*overridden.*")
-warnings.filterwarnings("ignore", message=".*unauthenticated.*")
-warnings.filterwarnings("ignore", message=".*UNEXPECTED.*")
-warnings.filterwarnings("ignore", category=UserWarning, module="umap")
-
-from sentence_transformers import SentenceTransformer
-from sklearn.metrics.pairwise import cosine_similarity as sk_cosine
-
-
-MODEL_NAME = "all-MiniLM-L6-v2"
-
-
-@lru_cache(maxsize=1)
-def get_embedder() -> SentenceTransformer:
-    """Lazy-load embedder. Cached so it only loads once per session."""
-    # Silence the load report printed to stdout by newer sentence-transformers
-    import io, contextlib
-    f = io.StringIO()
-    with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f):
-        model = SentenceTransformer(MODEL_NAME)
-    return model
-
-
-def embed(texts: list[str]) -> np.ndarray:
-    return get_embedder().encode(texts, show_progress_bar=False)
-
-
-def similarity(text_a: str, text_b: str) -> float:
-    """Cosine similarity between two texts. Returns float 0-1."""
-    embs = embed([text_a, text_b])
-    score = sk_cosine([embs[0]], [embs[1]])[0][0]
-    return float(np.clip(score, 0, 1))
-
-
-def batch_similarity(pairs: list[tuple[str, str]]) -> list[float]:
-    """
-    Efficient batch similarity for many pairs.
-    Embeds all texts in one pass instead of N passes.
-    """
-    all_texts = [t for pair in pairs for t in pair]
-    all_embs = embed(all_texts)
-    scores = []
-    for i in range(0, len(all_embs), 2):
-        score = sk_cosine([all_embs[i]], [all_embs[i + 1]])[0][0]
-        scores.append(float(np.clip(score, 0, 1)))
+"""
+Embedding + similarity layer.
+All local, all free. No API calls.
+"""
+from __future__ import annotations
+import logging
+import os
+import warnings
+import numpy as np
+from functools import lru_cache
+
+# Suppress noisy warnings from HuggingFace / sentence-transformers / UMAP
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
+logging.getLogger("transformers").setLevel(logging.ERROR)
+logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
+warnings.filterwarnings("ignore", message=".*n_jobs value.*overridden.*")
+warnings.filterwarnings("ignore", message=".*unauthenticated.*")
+warnings.filterwarnings("ignore", message=".*UNEXPECTED.*")
+warnings.filterwarnings("ignore", category=UserWarning, module="umap")
+
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity as sk_cosine
+
+
+MODEL_NAME = "all-MiniLM-L6-v2"
+
+
+@lru_cache(maxsize=1)
+def get_embedder() -> SentenceTransformer:
+    """Lazy-load embedder. Cached so it only loads once per session."""
+    # Silence the load report printed to stdout by newer sentence-transformers
+    import io, contextlib
+    f = io.StringIO()
+    with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f):
+        model = SentenceTransformer(MODEL_NAME)
+    return model
+
+
+def embed(texts: list[str]) -> np.ndarray:
+    return get_embedder().encode(texts, show_progress_bar=False)
+
+
+def similarity(text_a: str, text_b: str) -> float:
+    """Cosine similarity between two texts. Returns float 0-1."""
+    embs = embed([text_a, text_b])
+    score = sk_cosine([embs[0]], [embs[1]])[0][0]
+    return float(np.clip(score, 0, 1))
+
+
+def batch_similarity(pairs: list[tuple[str, str]]) -> list[float]:
+    """
+    Efficient batch similarity for many pairs.
+    Embeds all texts in one pass instead of N passes.
+    """
+    all_texts = [t for pair in pairs for t in pair]
+    all_embs = embed(all_texts)
+    scores = []
+    for i in range(0, len(all_embs), 2):
+        score = sk_cosine([all_embs[i]], [all_embs[i + 1]])[0][0]
+        scores.append(float(np.clip(score, 0, 1)))
     return scores