repobench 0.1.0__py3-none-any.whl

agentfit/__init__.py

@@ -0,0 +1,3 @@
+"""AgentFit — Living repository-native evals for coding agents."""
+
+__version__ = "0.1.0"

agentfit/analysis/metrics.py

@@ -0,0 +1,58 @@
+"""Configuration metrics aggregation from trials."""
+
+from __future__ import annotations
+
+import statistics
+from typing import Iterable
+
+from agentfit.logging import get_logger
+from agentfit.models import ConfigMetrics, Trial
+from agentfit.analysis.statistics import wilson_ci
+
+log = get_logger("analysis.metrics")
+
+
+def compute_config_metrics(trials: Iterable[Trial]) -> ConfigMetrics:
+    """Compute aggregated metrics for one agent configuration."""
+    trials = list(trials)
+    metrics = ConfigMetrics()
+
+    if not trials:
+        return metrics
+
+    solved = [t for t in trials if t.solved]
+    metrics.solved = len(solved)
+    metrics.total = len(trials)
+    metrics.pass_rate = len(solved) / len(trials)
+
+    # Wilson 95% CI
+    ci_lower, ci_upper = wilson_ci(len(solved), len(trials))
+    metrics.ci_lower = ci_lower
+    metrics.ci_upper = ci_upper
+
+    # Economics
+    costs = [t.cost_usd for t in trials if t.cost_usd is not None]
+    if costs:
+        metrics.total_cost = sum(costs)
+        metrics.mean_cost_task = metrics.total_cost / len(costs)
+        if metrics.solved > 0:
+            metrics.cost_per_solve = metrics.total_cost / metrics.solved
+
+    # Efficiency
+    prompt_tokens = [t.prompt_tokens for t in trials if t.prompt_tokens is not None]
+    completion_tokens = [t.completion_tokens for t in trials if t.completion_tokens is not None]
+    metrics.total_prompt_tokens = sum(prompt_tokens)
+    metrics.total_completion_tokens = sum(completion_tokens)
+    if metrics.solved > 0:
+        total_tokens = metrics.total_prompt_tokens + metrics.total_completion_tokens
+        metrics.tokens_per_solve = round(total_tokens / metrics.solved)
+
+    # Performance
+    durations = [t.duration_ms for t in trials if t.duration_ms is not None]
+    if durations:
+        metrics.p50_duration_ms = round(statistics.median(durations))
+        sorted_d = sorted(durations)
+        p90_idx = min(len(sorted_d) - 1, round(0.9 * (len(sorted_d) - 1)))
+        metrics.p90_duration_ms = sorted_d[p90_idx]
+
+    return metrics

agentfit/analysis/recommendation.py

@@ -0,0 +1,104 @@
+"""Cost-aware recommendation logic."""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from agentfit.logging import get_logger
+from agentfit.models import ConfigMetrics
+
+log = get_logger("analysis.recommendation")
+
+
+def recommend(
+    metrics_dict: dict[str, ConfigMetrics],
+) -> tuple[str | None, str]:
+    """Recommend a default agent configuration.
+
+    Policy (``cost_effective``):
+    1. Find the configuration with the highest observed pass rate.
+    2. Identify configurations whose quality difference is not
+       statistically conclusive (overlapping Wilson CIs).
+    3. Within that set, choose the lowest cost per verified solve.
+
+    Returns (config_name, reason).
+    """
+    if not metrics_dict:
+        return None, "No configurations with results available."
+
+    # Consider only configs with trials
+    with_results = {name: m for name, m in metrics_dict.items() if m.total > 0}
+    if not with_results:
+        return None, "No configurations have completed trials."
+
+    # 1. Best observed pass rate
+    best_name = max(with_results, key=lambda n: with_results[n].pass_rate)
+    best = with_results[best_name]
+
+    # 2. Statistically indistinguishable set (overlapping Wilson CIs)
+    indistinguishable = []
+    for name, m in with_results.items():
+        if _cis_overlap(best, m):
+            indistinguishable.append(name)
+
+    # 3. Lowest cost per verified solve among indistinguishable
+    with_cost = [
+        (name, with_results[name])
+        for name in indistinguishable
+        if with_results[name].cost_per_solve is not None
+    ]
+
+    if with_cost:
+        recommended = min(with_cost, key=lambda nm: nm[1].cost_per_solve)[0]
+        reason = (
+            f"lowest cost per verified solve among configurations "
+            f"statistically indistinguishable from observed best quality."
+        )
+    else:
+        # No cost data: pick best observed pass rate
+        recommended = best_name
+        reason = "best observed pass rate (no cost data available)."
+
+    log.info("Recommendation: %s (%s)", recommended, reason)
+    return recommended, reason
+
+
+def pareto_frontier(
+    metrics_dict: dict[str, ConfigMetrics],
+) -> list[str]:
+    """Identify configurations on the Pareto frontier.
+
+    A configuration is dominated if another config has >= pass rate AND
+    <= cost per solve (with at least one strict inequality).
+
+    Returns config names on the frontier (highest quality first).
+    """
+    with_cost = {
+        name: m for name, m in metrics_dict.items()
+        if m.total > 0 and m.cost_per_solve is not None
+    }
+    if not with_cost:
+        return list(metrics_dict.keys())
+
+    frontier: list[str] = []
+    for name, m in with_cost.items():
+        dominated = False
+        for other_name, other in with_cost.items():
+            if other_name == name:
+                continue
+            if (other.pass_rate >= m.pass_rate
+                    and other.cost_per_solve <= m.cost_per_solve
+                    and (other.pass_rate > m.pass_rate
+                         or other.cost_per_solve < m.cost_per_solve)):
+                dominated = True
+                break
+        if not dominated:
+            frontier.append(name)
+
+    frontier.sort(key=lambda n: with_cost[n].pass_rate, reverse=True)
+    return frontier
+
+
+def _cis_overlap(a: ConfigMetrics, b: ConfigMetrics) -> bool:
+    """Check if two Wilson CIs overlap (statistically indistinguishable)."""
+    return not (a.ci_upper < b.ci_lower or b.ci_upper < a.ci_lower)

agentfit/analysis/statistics.py

@@ -0,0 +1,71 @@
+"""Statistical utilities: Wilson CI and paired bootstrap."""
+
+from __future__ import annotations
+
+import math
+import random
+from typing import Iterable, Sequence
+
+from agentfit.logging import get_logger
+
+log = get_logger("analysis.statistics")
+
+_Z_95 = 1.959963984540054  # z-value for 95% confidence
+
+
+def wilson_ci(successes: int, total: int, z: float = _Z_95) -> tuple[float, float]:
+    """Wilson score interval for a binomial proportion.
+
+    Returns (lower, upper) bounds. If total is 0, returns (0, 0).
+    """
+    if total <= 0:
+        return 0.0, 0.0
+
+    p = successes / total
+    z2 = z * z
+    denom = 1 + z2 / total
+    center = (p + z2 / (2 * total)) / denom
+    margin = z * math.sqrt(p * (1 - p) / total + z2 / (4 * total * total)) / denom
+
+    return max(0.0, center - margin), min(1.0, center + margin)
+
+
+def paired_bootstrap_difference(
+    outcomes_a: Sequence[bool],
+    outcomes_b: Sequence[bool],
+    n_bootstrap: int = 10_000,
+    seed: int = 42,
+) -> tuple[float, float, float]:
+    """Paired bootstrap difference of pass rates.
+
+    Performs a paired bootstrap (resampling task indices with replacement)
+    with a deterministic seed.
+
+    Returns (observed_difference_pp, ci_lower_pp, ci_upper_pp) where
+    difference = pass_rate_a - pass_rate_b (in percentage points).
+    """
+    if len(outcomes_a) != len(outcomes_b):
+        raise ValueError("Paired bootstrap requires equal-length outcome arrays")
+
+    n = len(outcomes_a)
+    if n == 0:
+        return 0.0, 0.0, 0.0
+
+    a = [1 if x else 0 for x in outcomes_a]
+    b = [1 if x else 0 for x in outcomes_b]
+
+    observed_diff = (sum(a) - sum(b)) / n * 100
+
+    rng = random.Random(seed)
+    diffs: list[float] = []
+    for _ in range(n_bootstrap):
+        idx = [rng.randrange(n) for _ in range(n)]
+        sum_a = sum(a[i] for i in idx)
+        sum_b = sum(b[i] for i in idx)
+        diffs.append((sum_a - sum_b) / n * 100)
+
+    diffs.sort()
+    lo_idx = max(0, int(0.025 * n_bootstrap) - 1)
+    hi_idx = min(n_bootstrap - 1, int(0.975 * n_bootstrap) - 1)
+
+    return observed_diff, diffs[lo_idx], diffs[hi_idx]

agentfit/benchmark/coverage.py

@@ -0,0 +1,51 @@
+"""Coverage calculation: TVD converted to coverage percentages."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+
+def calculate_coverage(
+    benchmark_dist: dict[str, dict[str, float]],
+    workload_dist: dict[str, dict[str, float]],
+) -> dict[str, float]:
+    """Calculate coverage per dimension.
+
+    Args:
+        benchmark_dist: {dimension: {category: probability}}
+        workload_dist:  {dimension: {category: probability}}
+
+    Returns:
+        {dimension: coverage_percentage} where coverage = 100 * (1 - TVD).
+    """
+    coverage: dict[str, float] = {}
+
+    all_dims = set(benchmark_dist.keys()) | set(workload_dist.keys())
+    for dim in all_dims:
+        b = benchmark_dist.get(dim, {})
+        w = workload_dist.get(dim, {})
+        tvd = _tvd_from_dicts(b, w)
+        coverage[dim] = round(100 * (1 - tvd), 1)
+
+    return coverage
+
+
+def distributions_from_counts(
+    benchmark_counts: dict[str, Counter],
+    workload_counts: dict[str, Counter],
+) -> dict[str, float]:
+    """Convenience: compute coverage from Counter objects."""
+    b = {dim: {k: c / sum(c.values()) if sum(c.values()) else 0.0
+               for k, c in counts.items()} for dim, counts in benchmark_counts.items()}
+    w = {dim: {k: c / sum(c.values()) if sum(c.values()) else 0.0
+               for k, c in counts.items()} for dim, counts in workload_counts.items()}
+    return calculate_coverage(b, w)
+
+
+def _tvd_from_dicts(bench: dict[str, float], workload: dict[str, float]) -> float:
+    """Total Variation Distance between two probability dicts."""
+    keys = set(bench.keys()) | set(workload.keys())
+    if not keys:
+        return 0.0
+    return 0.5 * sum(abs(bench.get(k, 0.0) - workload.get(k, 0.0)) for k in keys)

agentfit/benchmark/health.py

@@ -0,0 +1,191 @@
+"""Benchmark Health calculation."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from agentfit.logging import get_logger
+from agentfit.models import BenchmarkHealth, CandidateTask, NetworkIsolation
+
+log = get_logger("benchmark.health")
+
+# Weights from the PRD
+_W_REPRESENTATIVENESS = 0.40
+_W_VALIDATION = 0.20
+_W_LEAKAGE = 0.15
+_W_RECENCY = 0.15
+_W_DIVERSITY = 0.10
+
+
+def calculate_health(
+    benchmark_tasks: list[CandidateTask],
+    workload_universe: list[CandidateTask] | None = None,
+) -> BenchmarkHealth:
+    """Compute the Benchmark Health score components.
+
+    Components (0-100 each):
+    - Representativeness: TVD-based coverage of workload distributions
+    - Validation: no-op/oracle/determinism validation strength
+    - Leakage: history sanitization, network isolation, credential removal
+    - Recency: temporal distribution of tasks
+    - Diversity: repetition of subsystem/complexity
+
+    Overall = weighted sum.
+    """
+    health = BenchmarkHealth()
+
+    if not benchmark_tasks:
+        return health
+
+    health.representativeness = _representativeness(benchmark_tasks, workload_universe)
+    health.validation = _validation(benchmark_tasks)
+    health.leakage = _leakage(benchmark_tasks)
+    health.recency = _recency(benchmark_tasks)
+    health.diversity = _diversity(benchmark_tasks)
+
+    health.overall = round(
+        _W_REPRESENTATIVENESS * health.representativeness
+        + _W_VALIDATION * health.validation
+        + _W_LEAKAGE * health.leakage
+        + _W_RECENCY * health.recency
+        + _W_DIVERSITY * health.diversity
+    )
+
+    return health
+
+
+def _representativeness(
+    tasks: list[CandidateTask],
+    workload: list[CandidateTask] | None,
+) -> int:
+    """Representativeness from distribution coverage (TVD-based)."""
+    if not workload:
+        return 100
+
+    from agentfit.benchmark.coverage import calculate_coverage
+
+    # Build distributions
+    def _dist(tasks_list: list[CandidateTask], key: str) -> dict[str, float]:
+        from collections import Counter
+        counts = Counter(getattr(t, key) for t in tasks_list)
+        total = sum(counts.values())
+        if total == 0:
+            return {}
+        return {k: v / total for k, v in counts.items()}
+
+    bench_dist = {
+        "task_type": _dist(tasks, "task_type"),
+        "subsystem": _dist(tasks, "subsystem"),
+        "complexity": _dist(tasks, "complexity"),
+    }
+    work_dist = {
+        "task_type": _dist(workload, "task_type"),
+        "subsystem": _dist(workload, "subsystem"),
+        "complexity": _dist(workload, "complexity"),
+    }
+
+    coverage = calculate_coverage(bench_dist, work_dist)
+    return round(sum(coverage.values()) / max(len(coverage), 1))
+
+
+def _validation(tasks: list[CandidateTask]) -> int:
+    """Validation confidence from eligibility flags."""
+    if not tasks:
+        return 0
+
+    scores = []
+    for t in tasks:
+        score = 100
+        e = t.eligibility
+
+        # No-op validated (must fail)
+        if getattr(e, "noop", None) is not True:
+            score -= 30
+        # Oracle validated (must pass)
+        if getattr(e, "oracle", None) is not True:
+            score -= 30
+        # Regression validated
+        if getattr(e, "regression", None) is not True:
+            score -= 10
+        # Determinism validated
+        if getattr(e, "determinism", None) is not True:
+            score -= 20
+        # Instruction provenance: Tier C reduces confidence
+        prov = getattr(t, "instruction_provenance", None)
+        if prov is not None and prov.value == "C":
+            score -= 15
+
+        scores.append(max(0, min(score, 100)))
+
+    return round(sum(scores) / len(scores))
+
+
+def _leakage(tasks: list[CandidateTask]) -> int:
+    """Leakage resistance score."""
+    if not tasks:
+        return 0
+
+    scores = []
+    for t in tasks:
+        score = 100
+        # Network isolation
+        net = getattr(t, "network_isolation", NetworkIsolation.NONE)
+        if net == NetworkIsolation.NONE:
+            score -= 25
+        elif net == NetworkIsolation.PARTIAL:
+            score -= 10
+
+        # Leakage risk from scanner
+        risk = getattr(t, "leakage_risk", 0.0)
+        score -= round(risk * 40)
+
+        # History eligibility (sanitized snapshot implies history available)
+        e = getattr(t, "eligibility", None)
+        if e is not None and getattr(e, "leakage", None) is True:
+            pass  # history sanitized
+        elif e is not None and getattr(e, "leakage", None) is False:
+            score -= 30
+
+        scores.append(max(0, min(score, 100)))
+
+    return round(sum(scores) / len(scores))
+
+
+def _recency(tasks: list[CandidateTask]) -> int:
+    """Recency score: how recent are the tasks (0-100)."""
+    now = datetime.now(timezone.utc)
+    scores = []
+    for t in tasks:
+        created = getattr(t, "created_at", None)
+        if created is None:
+            scores.append(50)
+            continue
+        if created.tzinfo is None:
+            created = created.replace(tzinfo=timezone.utc)
+        age_days = max(0, (now - created).days)
+        # 180-day window: recency decays linearly
+        score = max(0, 100 - age_days * 100 / 180)
+        scores.append(score)
+
+    if not scores:
+        return 0
+    return round(sum(scores) / len(scores))
+
+
+def _diversity(tasks: list[CandidateTask]) -> int:
+    """Diversity: penalize repeated subsystem/complexity combinations."""
+    if not tasks:
+        return 0
+
+    from collections import Counter
+    combos = Counter(
+        (t.subsystem or "unknown", t.complexity.value if t.complexity else "unknown")
+        for t in tasks
+    )
+    total = len(tasks)
+    max_repeat = max(combos.values())
+
+    # Ideal: all distinct (max_repeat=1). Score drops as max_repeat grows.
+    score = max(0, 100 - (max_repeat - 1) * 100 / max(total, 1))
+    return round(score)

agentfit/benchmark/sampling.py

@@ -0,0 +1,152 @@
+"""Representative benchmark sampling via greedy stratified optimization."""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+
+from agentfit.config import AgentFitConfig
+from agentfit.logging import get_logger
+from agentfit.models import CandidateTask, TaskStatus, TaskType, Complexity
+
+log = get_logger("benchmark.sampling")
+
+
+@dataclass
+class _Dist:
+    """A categorical distribution for one dimension."""
+
+    counts: Counter[str] = field(default_factory=Counter)
+
+    @property
+    def total(self) -> int:
+        return sum(self.counts.values())
+
+    def prob(self, key: str) -> float:
+        if self.total == 0:
+            return 0.0
+        return self.counts.get(key, 0) / self.total
+
+    def add(self, key: str) -> None:
+        self.counts[key] += 1
+
+
+def workload_distribution(
+    workload: list[CandidateTask],
+) -> tuple[_Dist, _Dist, _Dist]:
+    """Build task_type, subsystem, and complexity distributions."""
+    type_dist = _Dist()
+    sub_dist = _Dist()
+    comp_dist = _Dist()
+
+    for task in workload:
+        type_dist.add(task.task_type.value)
+        sub_dist.add(task.subsystem or "unknown")
+        comp_dist.add(task.complexity.value)
+
+    return type_dist, sub_dist, comp_dist
+
+
+def select_benchmark(
+    candidates: list[CandidateTask],
+    workload: list[CandidateTask],
+    config: AgentFitConfig,
+) -> list[CandidateTask]:
+    """Select a representative sample from VALID candidates.
+
+    Implements greedy stratified optimization: for each slot, pick the
+    candidate that most reduces the TVD between the benchmark distribution
+    and the workload distribution, with a diversity penalty.
+
+    Returns a list of selected candidates (up to config.benchmark.size).
+    """
+    valid = [c for c in candidates if c.status == TaskStatus.VALID]
+    if not valid:
+        log.warning("No VALID candidates available for sampling")
+        return []
+
+    size = config.benchmark.size
+    if size <= 0:
+        size = 24
+
+    size = min(size, len(valid))
+    weights = config.benchmark.dimensions
+
+    # Workload distributions (target)
+    w_type, w_sub, w_comp = workload_distribution(workload or valid)
+
+    # Benchmark distributions (current selection)
+    b_type = _Dist()
+    b_sub = _Dist()
+    b_comp = _Dist()
+
+    selected: list[CandidateTask] = []
+    available = list(valid)
+
+    for _ in range(size):
+        best_candidate = None
+        best_score = float("inf")
+
+        for cand in available:
+            # Simulate inclusion
+            sim_type = _Dist(b_type.counts.copy())
+            sim_sub = _Dist(b_sub.counts.copy())
+            sim_comp = _Dist(b_comp.counts.copy())
+            sim_type.add(cand.task_type.value)
+            sim_sub.add(cand.subsystem or "unknown")
+            sim_comp.add(cand.complexity.value)
+
+            # Total variation distance per dimension
+            tvd_type = _tvd(sim_type, w_type)
+            tvd_sub = _tvd(sim_sub, w_sub)
+            tvd_comp = _tvd(sim_comp, w_comp)
+
+            score = (
+                weights.task_type * tvd_type
+                + weights.subsystem * tvd_sub
+                + weights.complexity * tvd_comp
+            )
+
+            # Diversity penalty: penalize candidates too similar to selected
+            penalty = _diversity_penalty(cand, selected)
+            score += penalty
+
+            if score < best_score:
+                best_score = score
+                best_candidate = cand
+
+        if best_candidate is None:
+            break
+
+        selected.append(best_candidate)
+        available.remove(best_candidate)
+        b_type.add(best_candidate.task_type.value)
+        b_sub.add(best_candidate.subsystem or "unknown")
+        b_comp.add(best_candidate.complexity.value)
+
+    log.info("Selected %d tasks for benchmark (requested %d)", len(selected), size)
+    return selected
+
+
+def _tvd(bench: _Dist, workload: _Dist) -> float:
+    """Total Variation Distance between two categorical distributions."""
+    keys = set(bench.counts.keys()) | set(workload.counts.keys())
+    return 0.5 * sum(abs(bench.prob(k) - workload.prob(k)) for k in keys)
+
+
+def _diversity_penalty(candidate: CandidateTask, selected: list[CandidateTask]) -> float:
+    """Penalize candidates that are too similar to already-selected ones."""
+    if not selected:
+        return 0.0
+
+    penalty = 0.0
+    for sel in selected:
+        # Same PR family or same subsystem + complexity => high similarity
+        if sel.subsystem == candidate.subsystem and sel.complexity == candidate.complexity:
+            penalty += 0.10
+        if sel.task_type == candidate.task_type and sel.subsystem == candidate.subsystem:
+            penalty += 0.08
+        if sel.subsystem == candidate.subsystem:
+            penalty += 0.04
+
+    return penalty