mcpbr 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

mcpbr/__init__.py

@@ -3,7 +3,7 @@
 A benchmark runner for evaluating MCP servers against SWE-bench tasks.
 """
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"
 
 from .sdk import (
     BenchmarkResult,

mcpbr/analytics/__init__.py

@@ -0,0 +1,79 @@
+"""Analytics module for mcpbr historical results tracking and analysis.
+
+Provides:
+- SQLite-based historical results database
+- Time-series trend analysis
+- Statistical significance testing
+- Multi-model comparison engine
+- Error pattern analysis and anomaly detection
+- Correlation analysis and difficulty estimation
+- A/B testing framework
+- Leaderboard generation
+- Performance regression detection
+- Custom metrics registry
+"""
+
+from __future__ import annotations
+
+from .ab_testing import ABTest, run_ab_test
+from .anomaly import detect_anomalies, detect_metric_anomalies
+from .comparison import ComparisonEngine, compare_results_files, format_comparison_table
+from .correlation import (
+    analyze_metric_correlations,
+    find_strong_correlations,
+    pearson_correlation,
+    spearman_correlation,
+)
+from .database import ResultsDatabase
+from .difficulty import (
+    aggregate_difficulty_stats,
+    estimate_difficulty,
+    estimate_task_difficulty_score,
+)
+from .error_analysis import ErrorPatternAnalyzer, identify_flaky_tasks
+from .leaderboard import Leaderboard, generate_leaderboard
+from .metrics import MetricDefinition, MetricsRegistry
+from .regression_detector import RegressionDetector
+from .statistical import (
+    bootstrap_confidence_interval,
+    chi_squared_test,
+    compare_resolution_rates,
+    effect_size_cohens_d,
+    mann_whitney_u,
+    permutation_test,
+)
+from .trends import calculate_moving_average, calculate_trends, detect_trend_direction
+
+__all__ = [
+    "ABTest",
+    "ComparisonEngine",
+    "ErrorPatternAnalyzer",
+    "Leaderboard",
+    "MetricDefinition",
+    "MetricsRegistry",
+    "RegressionDetector",
+    "ResultsDatabase",
+    "aggregate_difficulty_stats",
+    "analyze_metric_correlations",
+    "bootstrap_confidence_interval",
+    "calculate_moving_average",
+    "calculate_trends",
+    "chi_squared_test",
+    "compare_resolution_rates",
+    "compare_results_files",
+    "detect_anomalies",
+    "detect_metric_anomalies",
+    "detect_trend_direction",
+    "effect_size_cohens_d",
+    "estimate_difficulty",
+    "estimate_task_difficulty_score",
+    "find_strong_correlations",
+    "format_comparison_table",
+    "generate_leaderboard",
+    "identify_flaky_tasks",
+    "mann_whitney_u",
+    "pearson_correlation",
+    "permutation_test",
+    "run_ab_test",
+    "spearman_correlation",
+]

mcpbr/analytics/ab_testing.py

@@ -0,0 +1,403 @@
+"""A/B testing framework for comparing MCP server configurations.
+
+Provides tools for statistically comparing two evaluation runs (control vs.
+treatment) to determine which configuration performs better on resolution rate,
+cost, and other metrics.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+def _normal_cdf(x: float) -> float:
+    """Approximate the standard normal cumulative distribution function.
+
+    Uses the Abramowitz & Stegun rational approximation (formula 26.2.17)
+    which is accurate to about 1e-5.
+
+    Args:
+        x: The z-score value.
+
+    Returns:
+        Probability that a standard normal variable is less than or equal to *x*.
+    """
+    if x < -8.0:
+        return 0.0
+    if x > 8.0:
+        return 1.0
+
+    sign = 1.0
+    if x < 0:
+        sign = -1.0
+        x = -x
+
+    t = 1.0 / (1.0 + 0.2316419 * x)
+    t2 = t * t
+    t3 = t2 * t
+    t4 = t3 * t
+    t5 = t4 * t
+
+    pdf = math.exp(-0.5 * x * x) / math.sqrt(2.0 * math.pi)
+    cdf_upper = pdf * (
+        0.319381530 * t - 0.356563782 * t2 + 1.781477937 * t3 - 1.821255978 * t4 + 1.330274429 * t5
+    )
+
+    if sign > 0:
+        return 1.0 - cdf_upper
+    else:
+        return cdf_upper
+
+
+def _chi_squared_test(
+    resolved_a: int,
+    total_a: int,
+    resolved_b: int,
+    total_b: int,
+) -> dict[str, Any]:
+    """Perform a chi-squared test for independence on two resolution rates.
+
+    Constructs a 2x2 contingency table of resolved/unresolved counts for
+    groups A and B and computes the chi-squared statistic with Yates'
+    continuity correction.  The p-value is derived from the chi-squared
+    distribution with 1 degree of freedom using a normal CDF approximation.
+
+    Args:
+        resolved_a: Number of resolved tasks in group A.
+        total_a: Total tasks in group A.
+        resolved_b: Number of resolved tasks in group B.
+        total_b: Total tasks in group B.
+
+    Returns:
+        Dictionary with ``chi_squared``, ``p_value``, ``significant``
+        (at alpha = 0.05), and ``degrees_of_freedom``.
+    """
+    unresolved_a = total_a - resolved_a
+    unresolved_b = total_b - resolved_b
+    grand_total = total_a + total_b
+
+    if grand_total == 0:
+        return {
+            "chi_squared": 0.0,
+            "p_value": 1.0,
+            "significant": False,
+            "degrees_of_freedom": 1,
+        }
+
+    # Expected values for the 2x2 table
+    total_resolved = resolved_a + resolved_b
+    total_unresolved = unresolved_a + unresolved_b
+
+    expected = [
+        [total_resolved * total_a / grand_total, total_unresolved * total_a / grand_total],
+        [total_resolved * total_b / grand_total, total_unresolved * total_b / grand_total],
+    ]
+
+    observed = [
+        [resolved_a, unresolved_a],
+        [resolved_b, unresolved_b],
+    ]
+
+    chi2 = 0.0
+    for i in range(2):
+        for j in range(2):
+            e = expected[i][j]
+            if e > 0:
+                # Yates' continuity correction
+                diff = abs(observed[i][j] - e) - 0.5
+                if diff < 0:
+                    diff = 0.0
+                chi2 += (diff * diff) / e
+
+    # Convert chi-squared (1 df) to p-value via normal approximation:
+    # If X ~ chi2(1), then sqrt(X) ~ N(0,1) approximately.
+    if chi2 > 0:
+        z = math.sqrt(chi2)
+        p_value = 2.0 * (1.0 - _normal_cdf(z))
+    else:
+        p_value = 1.0
+
+    return {
+        "chi_squared": round(chi2, 6),
+        "p_value": round(p_value, 6),
+        "significant": p_value < 0.05,
+        "degrees_of_freedom": 1,
+    }
+
+
+def _extract_metrics(results_data: dict[str, Any]) -> dict[str, Any]:
+    """Extract key metrics from a results_data dictionary.
+
+    Args:
+        results_data: Evaluation results with ``summary.mcp`` and ``tasks``.
+
+    Returns:
+        Dictionary with ``resolved``, ``total``, ``rate``, ``cost``,
+        ``model``, ``provider``, ``avg_tokens``, and ``avg_runtime``.
+    """
+    summary = results_data.get("summary", {}).get("mcp", {})
+    tasks = results_data.get("tasks", [])
+    metadata = results_data.get("metadata", {})
+    config = metadata.get("config", {})
+
+    resolved = summary.get("resolved", 0)
+    total = summary.get("total", 0)
+    rate = summary.get("rate", 0.0)
+    cost = summary.get("total_cost", 0.0)
+
+    total_tokens = 0
+    total_runtime = 0.0
+    task_count = len(tasks)
+
+    for task in tasks:
+        mcp = task.get("mcp", {})
+        tokens = mcp.get("tokens", {})
+        total_tokens += tokens.get("input", 0) + tokens.get("output", 0)
+        total_runtime += mcp.get("runtime_seconds", 0.0)
+
+    avg_tokens = total_tokens // task_count if task_count > 0 else 0
+    avg_runtime = total_runtime / task_count if task_count > 0 else 0.0
+
+    return {
+        "resolved": resolved,
+        "total": total,
+        "rate": rate,
+        "cost": cost,
+        "model": config.get("model", "unknown"),
+        "provider": config.get("provider", "unknown"),
+        "avg_tokens": avg_tokens,
+        "avg_runtime": round(avg_runtime, 2),
+    }
+
+
+class ABTest:
+    """A/B testing framework for comparing two MCP server configurations.
+
+    Creates a structured comparison between a control group (A) and treatment
+    group (B), running chi-squared significance testing on resolution rates
+    and comparing cost metrics.
+
+    Example::
+
+        test = ABTest("Model Comparison")
+        test.add_control(results_baseline)
+        test.add_treatment(results_candidate)
+        analysis = test.analyze()
+        print(test.format_report())
+    """
+
+    def __init__(
+        self,
+        name: str,
+        control_label: str = "A",
+        treatment_label: str = "B",
+    ) -> None:
+        """Initialize the A/B test.
+
+        Args:
+            name: Human-readable name for this test.
+            control_label: Label for the control group (default ``"A"``).
+            treatment_label: Label for the treatment group (default ``"B"``).
+        """
+        self.name = name
+        self.control_label = control_label
+        self.treatment_label = treatment_label
+        self._control: dict[str, Any] | None = None
+        self._treatment: dict[str, Any] | None = None
+        self._analysis: dict[str, Any] | None = None
+
+    def add_control(self, results_data: dict[str, Any]) -> None:
+        """Add the control group results.
+
+        Args:
+            results_data: Evaluation results dictionary for the control
+                configuration.
+        """
+        self._control = results_data
+        self._analysis = None
+
+    def add_treatment(self, results_data: dict[str, Any]) -> None:
+        """Add the treatment group results.
+
+        Args:
+            results_data: Evaluation results dictionary for the treatment
+                configuration.
+        """
+        self._treatment = results_data
+        self._analysis = None
+
+    def analyze(self) -> dict[str, Any]:
+        """Run the A/B test analysis.
+
+        Compares resolution rates using a chi-squared test, and reports
+        differences in cost and other metrics.
+
+        Returns:
+            Dictionary containing:
+                - ``test_name``: The test name.
+                - ``control``: Metrics for the control group.
+                - ``treatment``: Metrics for the treatment group.
+                - ``rate_difference``: Absolute difference in resolution rates.
+                - ``rate_relative_change``: Percentage change in resolution rate.
+                - ``cost_difference``: Difference in total cost.
+                - ``statistical_significance``: Chi-squared test results.
+                - ``winner``: ``"control"``, ``"treatment"``, or
+                  ``"no_significant_difference"``.
+                - ``recommendation``: Human-readable recommendation.
+
+        Raises:
+            ValueError: If control or treatment data has not been added.
+        """
+        if self._control is None:
+            raise ValueError("Control group results not set. Call add_control() first.")
+        if self._treatment is None:
+            raise ValueError("Treatment group results not set. Call add_treatment() first.")
+
+        ctrl = _extract_metrics(self._control)
+        treat = _extract_metrics(self._treatment)
+
+        rate_diff = treat["rate"] - ctrl["rate"]
+        rate_relative = (rate_diff / ctrl["rate"] * 100.0) if ctrl["rate"] > 0 else 0.0
+        cost_diff = treat["cost"] - ctrl["cost"]
+
+        significance = _chi_squared_test(
+            ctrl["resolved"],
+            ctrl["total"],
+            treat["resolved"],
+            treat["total"],
+        )
+
+        # Determine winner
+        if significance["significant"]:
+            if treat["rate"] > ctrl["rate"]:
+                winner = "treatment"
+            elif treat["rate"] < ctrl["rate"]:
+                winner = "control"
+            else:
+                winner = "no_significant_difference"
+        else:
+            winner = "no_significant_difference"
+
+        # Build recommendation
+        if winner == "treatment":
+            recommendation = (
+                f"Treatment ({self.treatment_label}) shows a statistically significant "
+                f"improvement of {rate_relative:+.1f}% in resolution rate. "
+                f"Recommend adopting the treatment configuration."
+            )
+        elif winner == "control":
+            recommendation = (
+                f"Control ({self.control_label}) performs significantly better. "
+                f"Treatment ({self.treatment_label}) shows a {rate_relative:+.1f}% change "
+                f"in resolution rate. Recommend keeping the control configuration."
+            )
+        else:
+            recommendation = (
+                f"No statistically significant difference detected between "
+                f"{self.control_label} and {self.treatment_label} "
+                f"(p={significance['p_value']:.4f}). Consider increasing sample size "
+                f"or testing with a larger effect."
+            )
+
+        self._analysis = {
+            "test_name": self.name,
+            "control": {
+                "label": self.control_label,
+                "resolved": ctrl["resolved"],
+                "total": ctrl["total"],
+                "rate": ctrl["rate"],
+                "cost": ctrl["cost"],
+            },
+            "treatment": {
+                "label": self.treatment_label,
+                "resolved": treat["resolved"],
+                "total": treat["total"],
+                "rate": treat["rate"],
+                "cost": treat["cost"],
+            },
+            "rate_difference": round(rate_diff, 6),
+            "rate_relative_change": round(rate_relative, 2),
+            "cost_difference": round(cost_diff, 4),
+            "statistical_significance": significance,
+            "winner": winner,
+            "recommendation": recommendation,
+        }
+
+        return self._analysis
+
+    def format_report(self) -> str:
+        """Format the analysis results as a human-readable report.
+
+        Calls :meth:`analyze` automatically if it has not been called yet.
+
+        Returns:
+            Multi-line string containing the formatted A/B test report.
+
+        Raises:
+            ValueError: If control or treatment data has not been added.
+        """
+        if self._analysis is None:
+            self.analyze()
+
+        assert self._analysis is not None  # for type checker
+        a = self._analysis
+
+        ctrl = a["control"]
+        treat = a["treatment"]
+        sig = a["statistical_significance"]
+
+        lines = [
+            f"{'=' * 60}",
+            f"A/B Test Report: {a['test_name']}",
+            f"{'=' * 60}",
+            "",
+            f"Control ({ctrl['label']}):",
+            f"  Resolution Rate: {ctrl['rate']:.1%} ({ctrl['resolved']}/{ctrl['total']})",
+            f"  Total Cost:      ${ctrl['cost']:.4f}",
+            "",
+            f"Treatment ({treat['label']}):",
+            f"  Resolution Rate: {treat['rate']:.1%} ({treat['resolved']}/{treat['total']})",
+            f"  Total Cost:      ${treat['cost']:.4f}",
+            "",
+            "Comparison:",
+            f"  Rate Difference:     {a['rate_difference']:+.4f} "
+            f"({a['rate_relative_change']:+.1f}%)",
+            f"  Cost Difference:     ${a['cost_difference']:+.4f}",
+            "",
+            "Statistical Significance:",
+            f"  Chi-squared: {sig['chi_squared']:.4f}",
+            f"  p-value:     {sig['p_value']:.6f}",
+            f"  Significant: {'Yes' if sig['significant'] else 'No'} (alpha=0.05)",
+            "",
+            f"Winner: {a['winner']}",
+            "",
+            f"Recommendation: {a['recommendation']}",
+            f"{'=' * 60}",
+        ]
+
+        return "\n".join(lines)
+
+
+def run_ab_test(
+    results_a: dict[str, Any],
+    results_b: dict[str, Any],
+    test_name: str = "A/B Test",
+) -> dict[str, Any]:
+    """Convenience function to run a quick A/B test comparison.
+
+    Creates an :class:`ABTest` instance, adds the control and treatment
+    data, and returns the analysis results.
+
+    Args:
+        results_a: Evaluation results for the control (A) group.
+        results_b: Evaluation results for the treatment (B) group.
+        test_name: Name for the test (default ``"A/B Test"``).
+
+    Returns:
+        Analysis dictionary from :meth:`ABTest.analyze`.
+    """
+    test = ABTest(test_name)
+    test.add_control(results_a)
+    test.add_treatment(results_b)
+    return test.analyze()

mcpbr/analytics/anomaly.py

@@ -0,0 +1,213 @@
+"""Anomaly detection for benchmark metrics.
+
+Provides statistical methods (z-score, IQR, MAD) to identify outlier
+values in benchmark cost, token, runtime, and iteration metrics.
+"""
+
+from __future__ import annotations
+
+import statistics
+from typing import Any
+
+
+def detect_anomalies(
+    values: list[float],
+    method: str = "zscore",
+    threshold: float = 2.0,
+) -> list[dict[str, Any]]:
+    """Detect anomalous values in a list of numeric measurements.
+
+    Args:
+        values: List of numeric values to analyse.
+        method: Detection method. One of ``"zscore"`` (z-score exceeds
+            threshold), ``"iqr"`` (IQR fence method), or ``"mad"``
+            (median absolute deviation). Defaults to ``"zscore"``.
+        threshold: Sensitivity threshold whose meaning depends on the
+            method. For ``"zscore"`` and ``"mad"`` it is the number of
+            standard deviations / MADs; for ``"iqr"`` it is the fence
+            multiplier (commonly 1.5). Defaults to 2.0.
+
+    Returns:
+        List of anomaly dicts, each containing:
+            - index: Position in the input list.
+            - value: The anomalous value.
+            - score: Computed score (z-score, IQR distance, or MAD score).
+            - method: The detection method used.
+
+    Raises:
+        ValueError: If *method* is not one of the supported methods.
+    """
+    dispatch = {
+        "zscore": _zscore_detect,
+        "iqr": _iqr_detect,
+        "mad": _mad_detect,
+    }
+    if method not in dispatch:
+        raise ValueError(f"Unknown method '{method}'. Supported: {', '.join(sorted(dispatch))}.")
+    return dispatch[method](values, threshold)
+
+
+def detect_metric_anomalies(
+    results_data: dict[str, Any],
+) -> dict[str, list[dict[str, Any]]]:
+    """Detect anomalies across standard benchmark metrics.
+
+    Extracts cost, total tokens, runtime, and iteration counts from the
+    results data and runs z-score anomaly detection on each metric.
+
+    Args:
+        results_data: Benchmark results dict with a ``tasks`` key
+            containing a list of task dicts. Each task should have an
+            ``mcp`` dict with ``cost``, ``tokens`` (with ``input`` and
+            ``output``), ``runtime_seconds``, and ``iterations``.
+
+    Returns:
+        Dict mapping metric name (``"cost"``, ``"tokens"``,
+        ``"runtime"``, ``"iterations"``) to a list of anomaly dicts
+        returned by :func:`detect_anomalies`.
+    """
+    tasks = results_data.get("tasks", [])
+    if not tasks:
+        return {
+            "cost": [],
+            "tokens": [],
+            "runtime": [],
+            "iterations": [],
+        }
+
+    costs: list[float] = []
+    tokens: list[float] = []
+    runtimes: list[float] = []
+    iterations: list[float] = []
+
+    for task in tasks:
+        mcp = task.get("mcp", {})
+        costs.append(float(mcp.get("cost", 0.0)))
+        token_info = mcp.get("tokens", {})
+        total_tokens = float(token_info.get("input", 0)) + float(token_info.get("output", 0))
+        tokens.append(total_tokens)
+        runtimes.append(float(mcp.get("runtime_seconds", 0.0)))
+        iterations.append(float(mcp.get("iterations", 0)))
+
+    return {
+        "cost": detect_anomalies(costs),
+        "tokens": detect_anomalies(tokens),
+        "runtime": detect_anomalies(runtimes),
+        "iterations": detect_anomalies(iterations),
+    }
+
+
+# ------------------------------------------------------------------
+# Detection method implementations
+# ------------------------------------------------------------------
+
+
+def _zscore_detect(
+    values: list[float],
+    threshold: float,
+) -> list[dict[str, Any]]:
+    """Detect anomalies using z-score.
+
+    A value is anomalous if its absolute z-score exceeds *threshold*.
+
+    Args:
+        values: Numeric values to analyse.
+        threshold: Z-score cutoff.
+
+    Returns:
+        List of anomaly dicts with index, value, score, and method.
+    """
+    if len(values) < 2:
+        return []
+
+    mean = statistics.mean(values)
+    stdev = statistics.pstdev(values)
+    if stdev == 0:
+        return []
+
+    anomalies: list[dict[str, Any]] = []
+    for i, v in enumerate(values):
+        z = abs(v - mean) / stdev
+        if z > threshold:
+            anomalies.append({"index": i, "value": v, "score": z, "method": "zscore"})
+    return anomalies
+
+
+def _iqr_detect(
+    values: list[float],
+    threshold: float,
+) -> list[dict[str, Any]]:
+    """Detect anomalies using the interquartile range (IQR) method.
+
+    Values outside ``[Q1 - threshold*IQR, Q3 + threshold*IQR]`` are
+    flagged. The score represents how many IQRs the value lies beyond
+    the nearest fence.
+
+    Args:
+        values: Numeric values to analyse.
+        threshold: IQR multiplier for the fence (commonly 1.5).
+
+    Returns:
+        List of anomaly dicts with index, value, score, and method.
+    """
+    if len(values) < 4:
+        return []
+
+    sorted_vals = sorted(values)
+    n = len(sorted_vals)
+    q1 = statistics.median(sorted_vals[: n // 2])
+    q3 = statistics.median(sorted_vals[(n + 1) // 2 :])
+    iqr = q3 - q1
+    if iqr == 0:
+        return []
+
+    lower_fence = q1 - threshold * iqr
+    upper_fence = q3 + threshold * iqr
+
+    anomalies: list[dict[str, Any]] = []
+    for i, v in enumerate(values):
+        if v < lower_fence:
+            score = (lower_fence - v) / iqr
+            anomalies.append({"index": i, "value": v, "score": score, "method": "iqr"})
+        elif v > upper_fence:
+            score = (v - upper_fence) / iqr
+            anomalies.append({"index": i, "value": v, "score": score, "method": "iqr"})
+    return anomalies
+
+
+def _mad_detect(
+    values: list[float],
+    threshold: float,
+) -> list[dict[str, Any]]:
+    """Detect anomalies using median absolute deviation (MAD).
+
+    The modified z-score is computed as ``0.6745 * (x - median) / MAD``.
+    Values whose absolute modified z-score exceeds *threshold* are
+    flagged.
+
+    Args:
+        values: Numeric values to analyse.
+        threshold: Modified z-score cutoff.
+
+    Returns:
+        List of anomaly dicts with index, value, score, and method.
+    """
+    if len(values) < 2:
+        return []
+
+    med = statistics.median(values)
+    abs_devs = [abs(v - med) for v in values]
+    mad = statistics.median(abs_devs)
+    if mad == 0:
+        return []
+
+    # 0.6745 is the 0.75th quantile of the standard normal distribution,
+    # used to make MAD consistent with standard deviation for normal data.
+    consistency_constant = 0.6745
+
+    anomalies: list[dict[str, Any]] = []
+    for i, v in enumerate(values):
+        modified_z = consistency_constant * abs(v - med) / mad
+        if modified_z > threshold:
+            anomalies.append({"index": i, "value": v, "score": modified_z, "method": "mad"})
+    return anomalies