codeprobe 0.5.2__tar.gz → 0.5.3__tar.gz

{codeprobe-0.5.2 → codeprobe-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeprobe
-Version: 0.5.2
+Version: 0.5.3
 Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
 Author: codeprobe contributors
 License-Expression: Apache-2.0

{codeprobe-0.5.2 → codeprobe-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codeprobe"
-version = "0.5.2"
+version = "0.5.3"
 description = "Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results."
 readme = "README.md"
 license = "Apache-2.0"

{codeprobe-0.5.2 → codeprobe-0.5.3}/src/codeprobe/analysis/stats.py

@@ -583,7 +583,31 @@ def compare_configs(
     elif speed_diff > 0:
         parts.append(f"{speed_diff:.1f}s slower")
 
-    summary = f"{a.label} vs {b.label}: {', '.join(parts)} " f"\u2192 {winner} wins"
+    # Soften the verdict when the effect is negligible or the test is
+    # underpowered, so we don't confidently declare a "winner" on what may
+    # be noise. Thresholds:
+    #   Cohen's d: |d| < 0.2 is "negligible" (Cohen 1988).
+    #   Cliff's delta: |delta| < 0.147 is "negligible" (Romano et al. 2006).
+    #   p-value > 0.05: not significant at the conventional threshold.
+    scores_tied = abs(score_diff) < 0.01
+    negligible_threshold = 0.2 if eff_method == "cohens_d" else 0.147
+    small_effect = (
+        eff_size is not None and abs(eff_size) < negligible_threshold
+    )
+    not_significant = p_val is not None and p_val > 0.05
+
+    if scores_tied:
+        verdict = "effectively tied"
+    elif small_effect and not_significant:
+        verdict = f"{winner} nominally ahead (not significant; small effect)"
+    elif small_effect:
+        verdict = f"{winner} nominally ahead (small effect size)"
+    elif not_significant:
+        verdict = f"{winner} nominally ahead (not significant at p=0.05)"
+    else:
+        verdict = f"{winner} wins"
+
+    summary = f"{a.label} vs {b.label}: {', '.join(parts)} \u2192 {verdict}"
 
     return PairwiseComparison(
         config_a=a.label,

{codeprobe-0.5.2 → codeprobe-0.5.3}/src/codeprobe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeprobe
-Version: 0.5.2
+Version: 0.5.3
 Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
 Author: codeprobe contributors
 License-Expression: Apache-2.0

{codeprobe-0.5.2 → codeprobe-0.5.3}/tests/test_stats.py

@@ -259,3 +259,60 @@ class TestComparePairwiseContinuousRouting:
             b_scores=[0.0, 1.0, 0.0],
         )
         assert cmp.effect_size_method == "cliffs_delta"
+
+
+class TestVerdictSoftening:
+    """Summary text softens the verdict when the effect is small or p is high."""
+
+    def _run_compare(self, a_scores, b_scores):
+        from codeprobe.analysis.stats import compare_configs
+        from codeprobe.models.experiment import ConfigResults
+
+        a_cr = ConfigResults(
+            config="a",
+            completed=[_task(f"t{i}", s) for i, s in enumerate(a_scores)],
+        )
+        b_cr = ConfigResults(
+            config="b",
+            completed=[_task(f"t{i}", s) for i, s in enumerate(b_scores)],
+        )
+        return compare_configs(
+            summarize_config(a_cr), summarize_config(b_cr),
+            a_scores=list(a_scores), b_scores=list(b_scores),
+        )
+
+    def test_large_effect_with_power_says_wins(self) -> None:
+        """Consistent large gap across enough samples → unqualified winner."""
+        # N=8, unambiguous separation in every paired sample
+        a = [0.90, 0.88, 0.92, 0.85, 0.87, 0.93, 0.89, 0.91]
+        b = [0.10, 0.12, 0.15, 0.08, 0.18, 0.11, 0.14, 0.09]
+        cmp = self._run_compare(a, b)
+        assert "a wins" in cmp.summary
+        assert "nominally" not in cmp.summary
+
+    def test_small_effect_softens_verdict(self) -> None:
+        """Noisy data with a tiny gap → softened verdict.
+
+        The gap (~0.02) clears the 0.01 tied threshold, but high within-
+        config variance keeps Cohen's d < 0.2, which should trigger the
+        "nominally ahead (small effect)" wording.
+        """
+        a = [0.95, 0.10, 0.85, 0.20, 0.75, 0.30]
+        b = [0.93, 0.08, 0.83, 0.18, 0.72, 0.28]
+        cmp = self._run_compare(a, b)
+        assert "nominally ahead" in cmp.summary
+        # Should NOT say "wins" unqualified
+        assert " a wins" not in cmp.summary
+        assert " b wins" not in cmp.summary
+
+    def test_tied_scores_report_tied(self) -> None:
+        cmp = self._run_compare([0.5, 0.5], [0.5, 0.5])
+        assert "effectively tied" in cmp.summary
+
+    def test_real_experiment_numbers_produce_softened_verdict(self) -> None:
+        """Regression: the kubernetes-mcp-comparison scenario (N=5, d=0.076)."""
+        baseline = [0.75, 0.40, 0.11, 0.71, 0.14]
+        with_mcp = [0.71, 0.36, 0.08, 0.71, 0.14]
+        cmp = self._run_compare(baseline, with_mcp)
+        # score_diff ~0.02, small cohen's d, high p → softened verdict
+        assert "nominally ahead" in cmp.summary