agent-regress-cli 0.1.0__py3-none-any.whl

agent_regress/__init__.py

@@ -0,0 +1,63 @@
+"""agent-regress: statistical regression testing for LLM agents.
+
+Find out if your latest agent deploy made things worse -- statistically.
+
+    from agent_regress import compare
+
+    report = compare(
+        version_a=my_agent_v1,
+        version_b=my_agent_v2,
+        test_suite=test_cases,
+        n_runs=50,
+    )
+    print(report)
+    report.assert_stable()
+"""
+
+from __future__ import annotations
+
+from agent_regress.ci.gate import RegressionGate, assert_no_regression
+from agent_regress.core.compare import compare
+from agent_regress.core.report import Report, Verdict
+from agent_regress.core.runner import (
+    AgentCallable,
+    AsyncAgentCallable,
+    ScorerCallable,
+    arun_suite,
+    concurrent_cancellation_probe,
+    run_suite,
+    subprocess_runner,
+)
+from agent_regress.core.scorer import (
+    exact_match_scorer,
+    f1_scorer,
+    no_path_leak_scorer,
+    schema_conformance_scorer,
+    state_diff_scorer,
+    structured_content_scorer,
+    tool_call_trace_scorer,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "AgentCallable",
+    "AsyncAgentCallable",
+    "RegressionGate",
+    "Report",
+    "ScorerCallable",
+    "Verdict",
+    "__version__",
+    "arun_suite",
+    "assert_no_regression",
+    "compare",
+    "concurrent_cancellation_probe",
+    "exact_match_scorer",
+    "f1_scorer",
+    "no_path_leak_scorer",
+    "run_suite",
+    "schema_conformance_scorer",
+    "state_diff_scorer",
+    "structured_content_scorer",
+    "subprocess_runner",
+    "tool_call_trace_scorer",
+]

agent_regress/benchmarks/__init__.py

@@ -0,0 +1,15 @@
+"""Standard benchmark harnesses: Tau-bench, GAIA, SWE-bench."""
+
+from agent_regress.benchmarks.gaia import GAIAHarness, GAIALevel, GAIALevelResult
+from agent_regress.benchmarks.swebench import SWEBenchHarness, SWEBenchResult
+from agent_regress.benchmarks.tau_bench import TauBenchHarness, TauBenchResult
+
+__all__ = [
+    "GAIAHarness",
+    "GAIALevel",
+    "GAIALevelResult",
+    "SWEBenchHarness",
+    "SWEBenchResult",
+    "TauBenchHarness",
+    "TauBenchResult",
+]

agent_regress/benchmarks/gaia.py

@@ -0,0 +1,80 @@
+"""GAIA Level 1-3 split harness.
+
+GAIA (General AI Assistants benchmark) has three difficulty levels.
+The Level 1-3 split reveals capability boundaries that aggregate scores hide.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Any
+
+
+class GAIALevel(IntEnum):
+    LEVEL_1 = 1
+    LEVEL_2 = 2
+    LEVEL_3 = 3
+
+
+@dataclass(frozen=True)
+class GAIALevelResult:
+    level: GAIALevel
+    accuracy: float
+    n_questions: int
+    n_correct: int
+
+
+@dataclass
+class GAIAHarness:
+    agent: Any
+    dataset: list[dict[str, Any]]
+
+    def _is_correct(self, output: Any, task: dict[str, Any]) -> bool:
+        if output is None:  # sentinel returned by _safe_run on exception
+            return False
+        expected = task.get("expected_answer", "")
+        return str(output).strip().lower() == str(expected).strip().lower()
+
+    def evaluate(self) -> list[GAIALevelResult]:
+        if not self.dataset:
+            raise ValueError("dataset must not be empty")
+
+        by_level: dict[GAIALevel, list[dict[str, Any]]] = {lv: [] for lv in GAIALevel}
+        for task in self.dataset:
+            raw_level = task.get("level", 1)
+            try:
+                level = GAIALevel(int(raw_level))
+            except (ValueError, TypeError):
+                level = GAIALevel.LEVEL_1
+            by_level[level].append(task)
+
+        results: list[GAIALevelResult] = []
+        for level, tasks in by_level.items():
+            if not tasks:
+                continue
+            correct = sum(
+                1 for task in tasks if self._is_correct(self._safe_run(task), task)
+            )
+            results.append(
+                GAIALevelResult(
+                    level=level,
+                    accuracy=correct / len(tasks),
+                    n_questions=len(tasks),
+                    n_correct=correct,
+                )
+            )
+        return sorted(results, key=lambda r: r.level)
+
+    def _safe_run(self, task: dict[str, Any]) -> Any:
+        try:
+            return self.agent(task)
+        except Exception as exc:
+            task_id = task.get("task_id", task.get("question_id", "unknown"))
+            warnings.warn(
+                f"Agent raised exception on task {task_id}: {exc}",
+                UserWarning,
+                stacklevel=2,
+            )
+            return None

agent_regress/benchmarks/swebench.py

@@ -0,0 +1,65 @@
+"""SWE-bench Verified scaffold score harness.
+
+Measures the framework contribution to SWE-bench pass rate independent
+of the underlying model. A better scaffold (test runner, output parser,
+verdict logic) can lift pass rates 5-15pp on the same model.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+from typing import Any
+
+_RESOLVED_STRINGS: frozenset[str] = frozenset({"resolved", "true", "pass", "1"})
+
+
+@dataclass(frozen=True)
+class SWEBenchResult:
+    scaffold_pass_rate: float
+    n_instances: int
+    n_resolved: int
+    instance_ids: list[str]
+    resolved_ids: list[str]
+
+
+@dataclass
+class SWEBenchHarness:
+    agent: Any
+    dataset: list[dict[str, Any]]
+
+    def _is_resolved(self, output: Any, instance: dict[str, Any]) -> bool:
+        if isinstance(output, bool):
+            return output
+        if isinstance(output, (int, float)):
+            return float(output) >= 0.5
+        return str(output).strip().lower() in _RESOLVED_STRINGS
+
+    def evaluate(self) -> SWEBenchResult:
+        if not self.dataset:
+            raise ValueError("dataset must not be empty")
+
+        resolved_ids: list[str] = []
+        all_ids: list[str] = []
+
+        for idx, instance in enumerate(self.dataset):
+            instance_id = str(instance.get("instance_id", f"instance_{idx}"))
+            all_ids.append(instance_id)
+            try:
+                output = self.agent(instance)
+                if self._is_resolved(output, instance):
+                    resolved_ids.append(instance_id)
+            except Exception as exc:
+                warnings.warn(
+                    f"Agent raised exception on {instance_id}: {exc}",
+                    UserWarning,
+                    stacklevel=2,
+                )
+
+        return SWEBenchResult(
+            scaffold_pass_rate=len(resolved_ids) / len(self.dataset),
+            n_instances=len(self.dataset),
+            n_resolved=len(resolved_ids),
+            instance_ids=all_ids,
+            resolved_ids=resolved_ids,
+        )

agent_regress/benchmarks/tau_bench.py

@@ -0,0 +1,86 @@
+"""Tau-bench pass^k harness.
+
+pass^k measures the probability that an agent succeeds on at least one
+attempt out of k independent runs. This captures reliability degradation
+that single-run benchmarks hide: an agent that succeeds 60% of the time
+will have pass^1=0.60, pass^4=0.974, pass^8=0.9993.
+
+Usage:
+    harness = TauBenchHarness(agent=my_agent, dataset=load_tau_bench())
+    result = harness.evaluate(k_values=[1, 4, 8])
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class TauBenchResult:
+    k: int
+    pass_at_k: float
+    n_tasks: int
+    n_successes: int
+    task_scores: list[float]
+
+
+@dataclass
+class TauBenchHarness:
+    agent: Any
+    dataset: list[dict[str, Any]]
+    success_threshold: float = 0.5
+
+    def _attempt(self, task: dict[str, Any]) -> bool:
+        try:
+            result = self.agent(task)
+            score = float(result) if isinstance(result, (int, float)) else 0.0
+            return score >= self.success_threshold
+        except Exception as exc:
+            warnings.warn(
+                f"Agent raised exception on task: {exc}",
+                UserWarning,
+                stacklevel=2,
+            )
+            return False
+
+    def _run_task_attempts(self, task: dict[str, Any], max_k: int) -> list[bool]:
+        results: list[bool] = []
+        for _ in range(max_k):
+            success = self._attempt(task)
+            results.append(success)
+            if success:
+                # any(results[:k]) == True for all k >= len(results) because
+                # Python slicing past the end returns all elements.
+                break
+        return results
+
+    def evaluate(self, k_values: list[int] | None = None) -> list[TauBenchResult]:
+        if k_values is None:
+            k_values = [1, 4, 8]
+        if not k_values:
+            raise ValueError("k_values must not be empty")
+        if not self.dataset:
+            raise ValueError("dataset must not be empty")
+
+        max_k = max(k_values)
+        # Run each task max_k times once; derive pass^k by checking first k attempts.
+        per_task_attempts = [
+            self._run_task_attempts(task, max_k) for task in self.dataset
+        ]
+
+        results: list[TauBenchResult] = []
+        for k in sorted(k_values):
+            task_passed = [any(attempts[:k]) for attempts in per_task_attempts]
+            n_success = sum(task_passed)
+            results.append(
+                TauBenchResult(
+                    k=k,
+                    pass_at_k=n_success / len(self.dataset),
+                    n_tasks=len(self.dataset),
+                    n_successes=n_success,
+                    task_scores=[1.0 if p else 0.0 for p in task_passed],
+                )
+            )
+        return results

agent_regress/ci/__init__.py

@@ -0,0 +1,5 @@
+"""CI gate for blocking deploys on statistically significant regressions."""
+
+from agent_regress.ci.gate import RegressionGate, assert_no_regression
+
+__all__ = ["RegressionGate", "assert_no_regression"]

agent_regress/ci/gate.py

@@ -0,0 +1,62 @@
+"""CI gate: raises AssertionError on statistically significant regression.
+
+Designed for use as a pytest test step. Import and call assert_no_regression()
+or use RegressionGate.check() inside a test function.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+from agent_regress.core.report import Report
+
+_MIN_N_WARNING = 30
+
+
+def assert_no_regression(
+    report: Report,
+    p_threshold: float = 0.05,
+    min_effect: float = 0.2,
+) -> None:
+    insufficient_n = report.n_a < _MIN_N_WARNING or report.n_b < _MIN_N_WARNING
+    if insufficient_n:
+        warnings.warn(
+            f"n_a={report.n_a}, n_b={report.n_b}: insufficient sample size for "
+            "reliable regression detection. Not failing the build due to "
+            "insufficient data. Use at least 30 runs per version.",
+            UserWarning,
+            stacklevel=2,
+        )
+        return
+
+    # Evaluate with the gate's own thresholds, not the verdict stored in the
+    # report (which was computed with compare()'s thresholds and may differ).
+    # report.assert_stable() also guards against INSUFFICIENT_DATA (n < 10).
+    report.assert_stable(p_threshold=p_threshold, min_effect=min_effect)
+
+
+class RegressionGate:
+    def __init__(
+        self,
+        p_threshold: float = 0.05,
+        min_effect: float = 0.2,
+    ) -> None:
+        if not (0.0 < p_threshold < 1.0):
+            raise ValueError(f"p_threshold must be in (0, 1), got {p_threshold}")
+        if min_effect < 0.0:
+            raise ValueError(f"min_effect must be >= 0, got {min_effect}")
+        self.p_threshold = p_threshold
+        self.min_effect = min_effect
+
+    def check(self, report: Report) -> None:
+        assert_no_regression(
+            report,
+            p_threshold=self.p_threshold,
+            min_effect=self.min_effect,
+        )
+
+    def __repr__(self) -> str:
+        return (
+            f"RegressionGate(p_threshold={self.p_threshold}, "
+            f"min_effect={self.min_effect})"
+        )

agent_regress/cli.py

@@ -0,0 +1,298 @@
+"""Command-line entry point for agent-regress.
+
+The public Python API (`agent_regress.compare()`) takes two *callables* --
+`version_a` and `version_b` -- and runs them itself via `run_suite()`. That
+shape has no clean shell equivalent: a CLI can't accept "a callable" as a
+flag value. So this CLI exposes a different, file-based contract that fits
+what a shell/agent workflow can realistically produce: two JSON files, each
+holding a flat array of per-run scores already computed by whatever harness
+ran that agent version (`[0.82, 0.79, 0.91, ...]`). Those two score arrays
+are fed through the exact same Mann-Whitney U / bootstrap CI / Cohen's d /
+Verdict pipeline `compare()` uses internally, so the statistical output is
+identical for the same underlying scores -- only how the scores get here
+differs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import warnings
+from pathlib import Path
+from typing import Any, NoReturn
+
+import numpy as np
+
+from agent_regress import __version__
+from agent_regress.core.report import Report, Verdict
+from agent_regress.stats.bootstrap import bootstrap_mean_ci
+from agent_regress.stats.effect_size import cohens_d
+from agent_regress.stats.mann_whitney import mann_whitney_u
+
+_MIN_N_WARN = 50
+_PROG = "agent-regress"
+
+
+def _fail(message: str) -> NoReturn:
+    """Print a usage/data error to stderr and exit non-zero.
+
+    Never writes to stdout, so `--json` invocations keep stdout clean even
+    on failure -- callers only need to check the exit code and stderr.
+    """
+    print(f"{_PROG}: error: {message}", file=sys.stderr)
+    raise SystemExit(2)
+
+
+def _load_scores(path: str, flag: str) -> list[float]:
+    """Load a JSON array of per-run scores from `path`.
+
+    Args:
+        path: Filesystem path to a JSON file containing a flat, non-empty
+            array of numbers, e.g. `[0.8, 0.9, 0.75]`.
+        flag: The CLI flag this path came from (e.g. "--version-a-results"),
+            used to make error messages actionable.
+
+    Returns:
+        The parsed scores as a list of floats.
+    """
+    file_path = Path(path)
+    if not file_path.is_file():
+        _fail(f"{flag}: no such file: {path}")
+    try:
+        raw = json.loads(file_path.read_text())
+    except json.JSONDecodeError as exc:
+        _fail(f"{flag}: {path} is not valid JSON ({exc})")
+    if not isinstance(raw, list) or not raw:
+        _fail(f"{flag}: {path} must contain a non-empty JSON array of scores")
+    try:
+        return [float(x) for x in raw]
+    except (TypeError, ValueError):
+        _fail(f"{flag}: {path} must contain only numbers, got: {raw!r}")
+
+
+def _report_from_scores(  # noqa: PLR0913
+    metric: str,
+    scores_a: list[float],
+    scores_b: list[float],
+    p_threshold: float,
+    min_effect: float,
+    n_resamples: int,
+) -> Report:
+    """Build a `Report` from two pre-computed score lists.
+
+    Mirrors the pipeline in `agent_regress.core.compare.compare()`
+    (Mann-Whitney U -> bootstrap CI -> Cohen's d -> Verdict thresholding),
+    minus the `run_suite()` step, since the CLI receives scores an external
+    harness already computed rather than callables it needs to invoke.
+    """
+    n_a, n_b = len(scores_a), len(scores_b)
+    warn_msgs: list[str] = []
+    if n_a < _MIN_N_WARN or n_b < _MIN_N_WARN:
+        msg = (
+            f"n_a={n_a}, n_b={n_b}: insufficient for 80% power at small effect "
+            f"(d=0.2). Run at least {_MIN_N_WARN} per version for reliable results."
+        )
+        warn_msgs.append(msg)
+        warnings.warn(msg, UserWarning, stacklevel=2)
+
+    mw = mann_whitney_u(scores_a, scores_b, _warn=False)
+    ci = bootstrap_mean_ci(scores_a, scores_b, n_resamples=n_resamples)
+    d = cohens_d(scores_a, scores_b)
+
+    arr_a = np.asarray(scores_a, dtype=np.float64)
+    arr_b = np.asarray(scores_b, dtype=np.float64)
+    std_a = float(arr_a.std(ddof=1)) if n_a > 1 else 0.0
+    std_b = float(arr_b.std(ddof=1)) if n_b > 1 else 0.0
+
+    if n_a < 10 or n_b < 10:
+        verdict = Verdict.INSUFFICIENT_DATA
+    elif mw.p_value < p_threshold and abs(d) >= min_effect:
+        verdict = Verdict.REGRESSED if d < 0.0 else Verdict.IMPROVED
+    else:
+        verdict = Verdict.STABLE
+
+    return Report(
+        metric=metric,
+        verdict=verdict,
+        p_value=mw.p_value,
+        effect_size=d,
+        ci_lower=ci.lower,
+        ci_upper=ci.upper,
+        n_a=n_a,
+        n_b=n_b,
+        mean_a=float(arr_a.mean()),
+        mean_b=float(arr_b.mean()),
+        std_a=std_a,
+        std_b=std_b,
+        mean_delta=ci.mean_delta,
+        p_threshold=p_threshold,
+        min_effect=min_effect,
+        warnings=warn_msgs,
+    )
+
+
+def _report_to_json(report: Report) -> dict[str, Any]:
+    """Flatten a `Report` into the same fields `str(report)` shows, as a
+    JSON-serializable dict -- the "agent-native" machine-readable surface.
+    """
+    return {
+        "metric": report.metric,
+        "verdict": report.verdict.value,
+        "p_value": report.p_value,
+        "effect_size": report.effect_size,
+        "ci_lower": report.ci_lower,
+        "ci_upper": report.ci_upper,
+        "n_a": report.n_a,
+        "n_b": report.n_b,
+        "mean_a": report.mean_a,
+        "mean_b": report.mean_b,
+        "std_a": report.std_a,
+        "std_b": report.std_b,
+        "mean_delta": report.mean_delta,
+        "p_threshold": report.p_threshold,
+        "min_effect": report.min_effect,
+        "warnings": report.warnings,
+    }
+
+
+def _cmd_compare(args: argparse.Namespace) -> int:
+    scores_a = _load_scores(args.version_a_results, "--version-a-results")
+    scores_b = _load_scores(args.version_b_results, "--version-b-results")
+
+    if args.n_resamples < 100:
+        _fail(f"--n-resamples must be >= 100, got {args.n_resamples}")
+    if not (0.0 < args.p_threshold < 1.0):
+        _fail(f"--p-threshold must be in (0, 1), got {args.p_threshold}")
+    if args.min_effect < 0.0:
+        _fail(f"--min-effect must be >= 0, got {args.min_effect}")
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        report = _report_from_scores(
+            metric=args.metric,
+            scores_a=scores_a,
+            scores_b=scores_b,
+            p_threshold=args.p_threshold,
+            min_effect=args.min_effect,
+            n_resamples=args.n_resamples,
+        )
+    # Warnings go to stderr only -- stdout stays clean JSON in --json mode.
+    for w in caught:
+        print(f"{_PROG}: warning: {w.message}", file=sys.stderr)
+
+    if args.json:
+        sys.stdout.write(json.dumps(_report_to_json(report)) + "\n")
+    else:
+        print(str(report))
+
+    if args.fail_on_regression and report.verdict == Verdict.REGRESSED:
+        return 1
+    return 0
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=_PROG,
+        description=(
+            "Statistical regression testing for LLM agents: compare two "
+            "sets of per-run scores and get a verdict (REGRESSED / STABLE / "
+            "IMPROVED / INSUFFICIENT_DATA) backed by a Mann-Whitney U test, "
+            "a bootstrap confidence interval, and Cohen's d effect size."
+        ),
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"{_PROG} {__version__}",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    compare_parser = subparsers.add_parser(
+        "compare",
+        help="Compare two versions' pre-computed per-run scores.",
+        description=(
+            "Compare two agent versions from pre-computed per-run scores. "
+            "Each --version-*-results file must be a JSON array of numbers, "
+            "one score per run, e.g. [0.82, 0.79, 0.91]. Produce these with "
+            "whatever harness ran each agent version -- the Python API's "
+            "compare() runs the harness for you via callables, but a CLI "
+            "invocation can't accept a callable, so this command starts "
+            "one step later, from the resulting scores."
+        ),
+    )
+    compare_parser.add_argument(
+        "--version-a-results",
+        required=True,
+        metavar="PATH",
+        help="Path to a JSON array of per-run scores for version A (baseline).",
+    )
+    compare_parser.add_argument(
+        "--version-b-results",
+        required=True,
+        metavar="PATH",
+        help="Path to a JSON array of per-run scores for version B (candidate).",
+    )
+    compare_parser.add_argument(
+        "--metric",
+        default="accuracy",
+        metavar="NAME",
+        help="Name of the metric being compared, shown in the report "
+        "(default: accuracy).",
+    )
+    compare_parser.add_argument(
+        "--p-threshold",
+        type=float,
+        default=0.05,
+        metavar="P",
+        help="Significance threshold for the Mann-Whitney U p-value (default: 0.05).",
+    )
+    compare_parser.add_argument(
+        "--min-effect",
+        type=float,
+        default=0.2,
+        metavar="D",
+        help="Minimum |Cohen's d| to call a statistically significant "
+        "difference a REGRESSED/IMPROVED verdict rather than STABLE "
+        "(default: 0.2).",
+    )
+    compare_parser.add_argument(
+        "--n-resamples",
+        type=int,
+        default=1000,
+        metavar="N",
+        help="Number of bootstrap resamples used for the confidence "
+        "interval (default: 1000, minimum: 100).",
+    )
+    compare_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Print the report as a single JSON object to stdout instead "
+        "of the human-readable format. Warnings go to stderr, so stdout "
+        "is clean, parseable JSON.",
+    )
+    compare_parser.add_argument(
+        "--fail-on-regression",
+        action="store_true",
+        help="Exit with status 1 if the verdict is REGRESSED (useful for "
+        "CI). Without this flag, the command exits 0 regardless of verdict, "
+        "matching how the Python compare() function never raises on its "
+        "own -- only Report.assert_stable() / the ci.gate module do.",
+    )
+    compare_parser.set_defaults(func=_cmd_compare)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help(sys.stderr)
+        return 2
+    result: int = args.func(args)
+    return result
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent_regress/core/__init__.py

@@ -0,0 +1,17 @@
+"""Core runner, scorer, and report types."""
+
+from agent_regress.core.compare import compare
+from agent_regress.core.report import Report, Verdict
+from agent_regress.core.runner import AgentCallable, ScorerCallable, run_suite
+from agent_regress.core.scorer import exact_match_scorer, f1_scorer
+
+__all__ = [
+    "AgentCallable",
+    "Report",
+    "ScorerCallable",
+    "Verdict",
+    "compare",
+    "exact_match_scorer",
+    "f1_scorer",
+    "run_suite",
+]