harness-evolve 0.1.0__py3-none-any.whl

harness_evolve/__init__.py

@@ -0,0 +1,20 @@
+"""
+harness-evolve: improve an agent's harness from its own failure traces.
+
+    from harness_evolve import AgentProfile, Optimizer, OptimizerConfig, profiles
+    profiles.use(AgentProfile.build(name=..., harness_root=..., agent_description=...))
+    opt = Optimizer(run_dir, base_harness, OptimizerConfig(evolve_cases=[...]),
+                    evaluator=MyEvaluator(), proposer_llm=llm, critic_llm=llm, critic_patterns=[])
+    state = asyncio.run(opt.run_until_stopped())
+"""
+from harness_evolve.acceptance import CaseResult, EvalResult
+from harness_evolve.evaluator import EvalOutcome, Evaluator, ProviderFailure, UnpricedModel
+from harness_evolve.harness import Harness, load_harness
+from harness_evolve.llm import LLMReply, litellm_llm
+from harness_evolve.loop import LocalPublisher, Optimizer, OptimizerConfig
+from harness_evolve.profiles import AgentProfile
+
+__all__ = ["AgentProfile", "CaseResult", "EvalOutcome", "EvalResult", "Evaluator", "Harness", "LLMReply",
+           "LocalPublisher", "Optimizer", "OptimizerConfig", "ProviderFailure", "UnpricedModel",
+           "litellm_llm", "load_harness"]
+__version__ = "0.1.0"

harness_evolve/acceptance.py

@@ -0,0 +1,237 @@
+"""
+Acceptance rules: when does a candidate harness replace the incumbent?
+
+The rules follow RRSI (Xia et al., "RRSI: Regularized Recursive
+Self-Improvement of Agent Harnesses", arXiv 2609.24972; reference
+implementation github.com/google-research/rrsi, Apache-2.0,
+rrsi/selection.py), reimplemented here for our setting:
+
+  floor       S' >= S* - delta                            (RRSI Eq. floor)
+  clear gain  if dS > delta:  dC <= beta0 + beta1 * dS    (RRSI Eq. tokenbudget)
+  in band     otherwise:      w_s*dS - w_c*dC > 0         (RRSI Alg. 2 l.5)
+  guards      domain vetoes (RRSI's eng domain vetoes a rising no-submission
+              rate; ours is the escalation rate, the same failure shape)
+
+dC is RELATIVE (C'/C - 1), as in RRSI. Defaults mirror RRSI's coding config,
+where w_s = 0: inside the noise band a candidate can only win by being
+cheaper, never by a within-noise score bump. Our noise is far larger than
+theirs (delta ~0.2-0.3 on ~17 cases vs their 0.017 on 178 trials), so in
+practice most gains are "in band" and the rule reduces to "no worse, and
+cheaper".
+
+What differs from RRSI, and why:
+
+- **Paired, per-case comparison instead of one aggregate score each.** The
+  candidate and incumbent run on the same cases, so S and C are compared
+  case by case, and each decision records which cases improved or
+  regressed. With ~20% of cases flipping between identical runs, knowing
+  *which* cases moved matters as much as the mean.
+- **Calibration from trial pairs.** RRSI estimates delta by re-running the
+  unchanged base harness. We get the same null distribution from round 0's
+  k=2 trials (trial 1 vs trial 2 of the same harness), bootstrapped over
+  cases: no extra run.
+- **Guard cases.** The evolve set's always-pass regression guards must keep
+  passing: a candidate that fails a guard on every trial (where the
+  incumbent passed it) is vetoed outright.
+- **No novelty term, pruning window or annealing.** Those need many rounds;
+  we can afford a handful. See docs/blog-drafts/harness-evolution-v2.md §0.4.
+"""
+from __future__ import annotations
+
+import random
+import statistics
+from dataclasses import asdict, dataclass, field
+
+
+@dataclass
+class CaseResult:
+    """One case's outcome across its trials under one harness."""
+    passes: int
+    trials: int
+    escalations: int = 0          # attempts that never got an accepted submission
+    cost_usd: float = 0.0         # summed over this case's trials
+    verdicts: list[str] = field(default_factory=list)
+
+    @property
+    def score(self) -> float:
+        return self.passes / self.trials if self.trials else 0.0
+
+
+@dataclass
+class EvalResult:
+    """A harness evaluated on a set of cases. S and C are means over cases."""
+    per_case: dict[str, CaseResult]
+
+    @property
+    def S(self) -> float:
+        return statistics.fmean(c.score for c in self.per_case.values()) if self.per_case else 0.0
+
+    @property
+    def C(self) -> float:
+        """Mean cost per trial."""
+        trials = sum(c.trials for c in self.per_case.values())
+        return sum(c.cost_usd for c in self.per_case.values()) / trials if trials else 0.0
+
+    @property
+    def escalation_rate(self) -> float:
+        trials = sum(c.trials for c in self.per_case.values())
+        return sum(c.escalations for c in self.per_case.values()) / trials if trials else 0.0
+
+    def to_json(self) -> dict:
+        return {"S": self.S, "C": self.C, "escalation_rate": self.escalation_rate,
+                "per_case": {k: asdict(v) for k, v in self.per_case.items()}}
+
+    @classmethod
+    def from_json(cls, data: dict) -> EvalResult:
+        return cls({k: CaseResult(**v) for k, v in data["per_case"].items()})
+
+
+@dataclass(frozen=True)
+class AcceptanceConfig:
+    delta: float = 0.25            # noise band; replaced by calibrate() after round 0
+    beta0: float = 0.10            # RRSI coding: 10% more cost for free on a clear gain
+    beta1: float = 44.5            # RRSI coding: each +1pp of S buys +44.5% cost
+    w_s: float = 0.0               # RRSI coding: no credit for within-band score change
+    w_c: float = 15.0              # RRSI coding
+    max_escalation_rise: float = 0.05
+    guard_cases: tuple[str, ...] = ()
+    # Noise band on the relative cost change, calibrated like delta from round
+    # 0's trial pairs. 0 = the original rule (in band, any saving counts).
+    cost_delta: float = 0.0
+    # Compare cost only on cases both harnesses resolved on every trial, so a
+    # cheaper failure never counts as a saving (multi-harness RL article: reward
+    # efficiency only on correct answers). Off = cost over all shared cases.
+    cost_on_shared_successes: bool = False
+    min_shared_successes: int = 5
+    # A rejected in-band candidate with dS >= this fraction of delta is kept as a
+    # near miss the proposer can build on (DGM/HGM archives). 0 = off.
+    near_miss_fraction: float = 0.0
+
+
+@dataclass
+class Decision:
+    accept: bool
+    reason: str
+    delta_S: float
+    delta_C: float                                  # relative
+    improved: list[str] = field(default_factory=list)
+    regressed: list[str] = field(default_factory=list)
+    vetoes: list[str] = field(default_factory=list)
+    near_miss: bool = False
+
+    def to_json(self) -> dict:
+        return asdict(self)
+
+
+def paired_deltas(incumbent: EvalResult, candidate: EvalResult,
+                  cfg: AcceptanceConfig | None = None) -> tuple[float, float, list, list]:
+    """dS over the cases both evaluated, relative dC, and which cases moved.
+    With cfg.cost_on_shared_successes, dC is over the cases both resolved on
+    every trial (0 if fewer than cfg.min_shared_successes)."""
+    shared = sorted(set(incumbent.per_case) & set(candidate.per_case))
+    if not shared:
+        raise ValueError("candidate and incumbent share no evaluated cases")
+    inc = EvalResult({c: incumbent.per_case[c] for c in shared})
+    cand = EvalResult({c: candidate.per_case[c] for c in shared})
+    dS = cand.S - inc.S
+    dC = (cand.C / inc.C - 1.0) if inc.C > 0 else 0.0
+    if cfg is not None and cfg.cost_on_shared_successes:
+        both = [c for c in shared if inc.per_case[c].score == 1.0 and cand.per_case[c].score == 1.0]
+        if len(both) < cfg.min_shared_successes:
+            dC = 0.0
+        else:
+            inc_s = EvalResult({c: inc.per_case[c] for c in both})
+            cand_s = EvalResult({c: cand.per_case[c] for c in both})
+            dC = (cand_s.C / inc_s.C - 1.0) if inc_s.C > 0 else 0.0
+    improved = [c for c in shared if cand.per_case[c].score > inc.per_case[c].score]
+    regressed = [c for c in shared if cand.per_case[c].score < inc.per_case[c].score]
+    return dS, dC, improved, regressed
+
+
+def decide(incumbent: EvalResult, candidate: EvalResult, S_star: float,
+           cfg: AcceptanceConfig) -> Decision:
+    dS, dC, improved, regressed = paired_deltas(incumbent, candidate, cfg)
+    d = Decision(False, "", dS, dC, improved, regressed)
+
+    # A guard vetoes only when it fails EVERY trial of the candidate and the
+    # incumbent passed it at least once. The first rule (any failed trial)
+    # was noise at 2 trials a case: a guard that passes 80-95% of the time
+    # (astropy-14309 went PASS/FAIL on the unchanged harness) would have
+    # vetoed about a third of candidates by chance across 4 guards. Breaking
+    # the easy path shows up as all-fail, not one-in-two.
+    for g in cfg.guard_cases:
+        cr = candidate.per_case.get(g)
+        inc = incumbent.per_case.get(g)
+        if cr is not None and cr.trials and cr.passes == 0 and (inc is None or inc.passes > 0):
+            d.vetoes.append(f"guard case {g} failed {cr.trials}/{cr.trials}")
+    rise = candidate.escalation_rate - incumbent.escalation_rate
+    if rise > cfg.max_escalation_rise:
+        d.vetoes.append(f"escalation rate rose {rise:+.3f} (limit {cfg.max_escalation_rise})")
+    if d.vetoes:
+        d.reason = "vetoed: " + "; ".join(d.vetoes)
+        return d
+
+    floor = S_star - cfg.delta
+    if candidate.S < floor:
+        d.reason = f"below noise floor: S' {candidate.S:.3f} < S* {S_star:.3f} - delta {cfg.delta:.3f}"
+        return d
+
+    if dS > cfg.delta:
+        budget = cfg.beta0 + cfg.beta1 * dS
+        d.accept = dC <= budget
+        d.reason = (f"clear gain dS {dS:+.3f} > delta {cfg.delta:.3f}; relative cost "
+                    f"{dC:+.3f} {'<=' if d.accept else '>'} budget {budget:.3f}")
+        return d
+
+    # In band a candidate must be cheaper, and with a cost band, cheaper by
+    # more than cost noise: the same "submit sooner" edit measured dC anywhere
+    # from -1.8% to +4.8% across evaluations, so comparing with 0 let noise decide.
+    shaped = cfg.w_s * dS - cfg.w_c * (dC + cfg.cost_delta)
+    d.accept = shaped > 0
+    d.near_miss = (not d.accept and cfg.near_miss_fraction > 0 and dS > 0
+                   and dS >= cfg.near_miss_fraction * cfg.delta)
+    band = f", cost band {cfg.cost_delta:.3f}" if cfg.cost_delta else ""
+    d.reason = (f"within noise band (dS {dS:+.3f}, delta {cfg.delta:.3f}{band}); "
+                f"{cfg.w_s}*dS - {cfg.w_c}*(dC{' + band' if cfg.cost_delta else ''}) = {shaped:+.3f} "
+                f"{'>' if d.accept else '<='} 0"
+                + ("" if d.accept else " (in band, a candidate must be cheaper"
+                   + (" by more than cost noise)" if cfg.cost_delta else ")")))
+    return d
+
+
+def calibrate_cost_delta(trial_costs: dict[str, tuple[float, float]], z: float = 2.0,
+                         trials_per_eval: int = 1, reps: int = 2000, seed: int = 7) -> float:
+    """Noise band on the RELATIVE cost change, from two trials of the same
+    harness per case: the null dC is (sum trial1 - sum trial2) / mean total,
+    bootstrapped over cases, times z, shrunk by sqrt(trials_per_eval)."""
+    pairs = list(trial_costs.values())
+    if len(pairs) < 2:
+        raise ValueError("need at least 2 cases with two trials each")
+    rng = random.Random(seed)
+    boot = []
+    for _ in range(reps):
+        sample = [rng.choice(pairs) for _ in pairs]
+        a, b = sum(x for x, _ in sample), sum(y for _, y in sample)
+        boot.append((a - b) / ((a + b) / 2) if a + b else 0.0)
+    return z * statistics.pstdev(boot) / trials_per_eval ** 0.5
+
+
+def calibrate_delta(trial_pairs: dict[str, tuple[bool, bool]], z: float = 2.0,
+                    trials_per_eval: int = 1, reps: int = 2000, seed: int = 7) -> float:
+    """Noise band from two trials of the SAME harness on each case.
+
+    The null difference of two independent evaluations is observed directly:
+    dS_null = mean over cases of (trial1 - trial2). Its standard deviation is
+    bootstrapped over cases, and delta = z * sd, so an unchanged harness
+    clears the floor about 97.5% of the time at z = 2 (as in rrsi/calibrate.py).
+
+    The pairs measure single-trial evaluations. If each later evaluation
+    averages `trials_per_eval` trials per case, its noise shrinks by
+    sqrt(trials_per_eval), and so does delta.
+    """
+    diffs = [float(a) - float(b) for a, b in trial_pairs.values()]
+    if len(diffs) < 2:
+        raise ValueError("need at least 2 cases with two trials each to calibrate")
+    rng = random.Random(seed)
+    boot = [statistics.fmean(rng.choice(diffs) for _ in diffs) for _ in range(reps)]
+    return z * statistics.pstdev(boot) / trials_per_eval ** 0.5

harness_evolve/bench.py

@@ -0,0 +1,137 @@
+"""
+Benchmark the optimizer itself, so "this change makes harness-evolve better" is a measured claim.
+
+A benchmark is a list of scenarios. Each builds an Optimizer in its own run directory and may state
+what must happen:
+
+- Known-answer scenarios: a scripted proposer (and critic) puts one known edit in front of the
+  acceptance rules, so there is a right answer. A real fix to a planted flaw must be accepted (the
+  rules can see a real gain); a harmful edit must be rejected; a neutral rewording must not be
+  accepted (false accepts).
+- Open scenarios: the real proposer, reported but not pass/fail: does it find a planted flaw, and
+  what does a standard run gain on held-out cases, per dollar?
+
+    harness-evolve bench examples/math_agent/bench.py --out runs/bench-1
+
+Run directories persist, so a stopped benchmark resumes like any run. Results go to
+<out>/bench.json and <out>/bench.md.
+"""
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from harness_evolve.llm import LLM, LLMReply
+
+ACCEPTED, NOT_ACCEPTED = "accepted", "not_accepted"
+
+
+@dataclass
+class Scenario:
+    name: str
+    description: str
+    build: Callable[[Path], object]          # run_dir -> Optimizer
+    expect: str | None = None                # ACCEPTED, NOT_ACCEPTED, or None (report only)
+
+
+def scripted_llm(edit: dict, critic_accepts: bool = True) -> LLM:
+    """A stand-in for the proposer and critic: the proposer always proposes `edit`
+    ({"component", "hypothesis", "edits": [...]}); the critic always accepts (or rejects).
+    With the critic scripted too, the scenario tests the acceptance rules alone."""
+    async def call(system: str, prompt: str) -> LLMReply:
+        if "strict reviewer" in system:
+            verdict = "accept" if critic_accepts else "reject"
+            return LLMReply(json.dumps({"verdict": verdict, "reasons": ["scripted"]}), 0.0)
+        return LLMReply(json.dumps(edit), 0.0)
+    return call
+
+
+def outcome(run_dir: Path) -> dict:
+    """What a finished scenario run did: rounds, acceptances, held-out numbers, spend."""
+    state = json.loads((run_dir / "state.json").read_text())
+    rounds = []
+    hist = run_dir / "history.jsonl"
+    if hist.exists():
+        for line in hist.read_text().splitlines():
+            e = json.loads(line)
+            if not str(e.get("outcome", "")).startswith("pilot_"):
+                rounds.append({k: e.get(k) for k in ("round", "component", "outcome", "delta_S", "delta_C",
+                                                     "reason", "hypothesis")})
+    out = {"phase": state.get("phase"), "stop_reason": state.get("stop_reason"),
+           "accepted": state.get("accepted") or [], "spent_usd": round(state.get("spent_usd") or 0.0, 3),
+           "S_star": state.get("S_star"), "delta": state.get("delta"), "rounds": rounds}
+    rep = run_dir / "report" / "report.json"
+    if rep.exists():
+        r = json.loads(rep.read_text())
+        out["heldout"] = {
+            "original_pass@1": (r.get("original") or {}).get("pass@1"),
+            "evolved_pass@1": (r.get("evolved") or {}).get("pass@1"),
+            "vs_original_single_run": (r.get("comparisons") or {}).get("vs_original_single_run"),
+            "vs_rerun_baseline_best_of_2": (r.get("comparisons") or {}).get("vs_rerun_baseline_best_of_2"),
+        }
+    return out
+
+
+def judge(expect: str | None, result: dict) -> bool | None:
+    if result.get("phase") != "done":
+        return False
+    if expect == ACCEPTED:
+        return bool(result["accepted"])
+    if expect == NOT_ACCEPTED:
+        return not result["accepted"]
+    return None
+
+
+async def run(scenarios: list[Scenario], out_dir: Path, only: list[str] | None = None) -> list[dict]:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    results = []
+    for sc in scenarios:
+        if only and sc.name not in only:
+            continue
+        run_dir = out_dir / sc.name
+        opt = sc.build(run_dir)
+        try:
+            await opt.run_until_stopped()
+        except Exception as exc:                       # one broken scenario shouldn't hide the rest
+            results.append({"name": sc.name, "description": sc.description, "expect": sc.expect,
+                            "passed": False, "error": f"{type(exc).__name__}: {exc}"})
+            continue
+        res = outcome(run_dir)
+        results.append({"name": sc.name, "description": sc.description, "expect": sc.expect,
+                        "passed": judge(sc.expect, res), **res})
+    if only and (out_dir / "bench.json").exists():
+        # A partial re-run updates its scenarios in the existing results instead of replacing them.
+        previous = json.loads((out_dir / "bench.json").read_text())
+        fresh = {r["name"]: r for r in results}
+        names = [r["name"] for r in previous] + [n for n in fresh if n not in {r["name"] for r in previous}]
+        old = {r["name"]: r for r in previous}
+        results = [fresh.get(n, old.get(n)) for n in names]
+    (out_dir / "bench.json").write_text(json.dumps(results, indent=1))
+    (out_dir / "bench.md").write_text(render(results))
+    return results
+
+
+def render(results: list[dict]) -> str:
+    def pct(x):
+        return "-" if x is None else f"{x:.3f}"
+    lines = ["# harness-evolve benchmark", "",
+             "| Scenario | Expect | Result | Accepted | First round (dS, dC) "
+             "| Held-out pass@1 (orig → evolved) | Spent |",
+             "|---|---|---|---|---|---|---|"]
+    for r in results:
+        verdict = {True: "PASS", False: "FAIL", None: "report"}[r.get("passed")]
+        if r.get("error"):
+            lines.append(f"| {r['name']} | {r.get('expect') or '-'} | ERROR | - | {r['error'][:80]} | - | - |")
+            continue
+        first = next(iter(r.get("rounds") or []), None)
+        fr = (f"{first['outcome']} ({first['delta_S']:+.3f}, {first['delta_C']:+.0%})"
+              if first and first.get("delta_S") is not None and first.get("delta_C") is not None
+              else (first or {}).get("outcome", "-"))
+        h = r.get("heldout") or {}
+        ho = (f"{pct(h.get('original_pass@1'))} → {pct(h.get('evolved_pass@1'))}" if h else "-")
+        lines.append(f"| {r['name']} | {r.get('expect') or '-'} | {verdict} | {r.get('accepted') or 'none'} | "
+                     f"{fr} | {ho} | ${r.get('spent_usd', 0):.2f} |")
+    lines += ["", *[f"- **{r['name']}**: {r['description']}" for r in results]]
+    return "\n".join(lines) + "\n"

harness_evolve/budget.py

@@ -0,0 +1,57 @@
+"""
+Hard spending cap, enforced in code, never left to the model.
+
+Before an evaluation starts, the optimizer asks for a reservation sized by
+the expected cost (per-case cost measured so far x cases x trials, with a
+safety margin). If the reservation would take total spend past the cap, the
+evaluation doesn't start and the run stops with a clear reason, rather than
+discovering the overrun (or an empty credit balance, twice this week) halfway
+through. Actual spend comes from the cost meter and is written into the run
+state, so a resumed run knows what it has already spent.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class BudgetExceeded(RuntimeError):
+    pass
+
+
+@dataclass
+class Budget:
+    cap_usd: float
+    spent_usd: float = 0.0
+    margin: float = 1.25            # reserve 25% above the estimate
+    # Reservations held by evaluations still running. With several running in
+    # parallel, each checking only `spent` could pass the cap together and
+    # overshoot it; counting in-flight reservations prevents that.
+    reserved_usd: float = 0.0
+
+    @property
+    def remaining_usd(self) -> float:
+        return max(0.0, self.cap_usd - self.spent_usd - self.reserved_usd)
+
+    def reserve(self, estimate_usd: float, what: str) -> float:
+        """Hold `estimate x margin` until release(); returns the amount held."""
+        need = estimate_usd * self.margin
+        if self.spent_usd + self.reserved_usd + need > self.cap_usd:
+            raise BudgetExceeded(
+                f"{what}: needs ~${need:.2f} (estimate ${estimate_usd:.2f} x {self.margin} margin), "
+                f"but only ${self.remaining_usd:.2f} of the ${self.cap_usd:.2f} cap remains"
+                + (f" (${self.reserved_usd:.2f} held by evaluations in flight)" if self.reserved_usd else "")
+            )
+        self.reserved_usd += need
+        return need
+
+    def release(self, held_usd: float, actual_usd: float) -> None:
+        """Swap a reservation for what the evaluation actually cost."""
+        self.reserved_usd = max(0.0, self.reserved_usd - held_usd)
+        self.spent_usd += actual_usd
+
+    def record(self, actual_usd: float) -> None:
+        self.spent_usd += actual_usd
+
+
+def estimate_eval_cost(n_cases: int, trials: int, per_case_usd: float) -> float:
+    return n_cases * trials * per_case_usd

harness_evolve/candidates.py

@@ -0,0 +1,112 @@
+"""
+Candidate harnesses: copies of a harness directory with one edit applied.
+
+The edit surface is bounded to the files that already exist in the harness
+directory: an edit may rewrite them, never add or delete files. Before anything is spent
+evaluating a candidate, it must pass
+structural validation, so a candidate that can't even load (bad JSON, a
+template placeholder the code never fills, a removed tool description, a
+turn budget of 500) is rejected for free. RRSI does the same with its
+"deterministic compile/constructor/smoke checks" after the critic.
+"""
+from __future__ import annotations
+
+import difflib
+import hashlib
+import json
+import shutil
+import string
+from pathlib import Path
+
+IGNORED = {"README.md"}         # documentation, never loaded, never edited
+
+class InvalidCandidate(ValueError):
+    pass
+
+
+def harness_files(directory: Path) -> dict[str, str]:
+    return {p.name: p.read_text() for p in sorted(Path(directory).iterdir())
+            if p.is_file() and p.name not in IGNORED}
+
+
+def content_hash(directory: Path) -> str:
+    h = hashlib.sha256()
+    for name, text in harness_files(directory).items():
+        h.update(name.encode() + b"\0" + text.encode() + b"\0")
+    return h.hexdigest()[:16]
+
+
+def _placeholders(template: str) -> set[str]:
+    return {f for _, f, _, _ in string.Formatter().parse(template) if f}
+
+
+def apply_edit(base: Path, dest: Path, files: dict[str, str]) -> None:
+    """dest = copy of base, with `files` (name -> full new content) overwritten."""
+    base_files = harness_files(base)
+    unknown = sorted(set(files) - set(base_files))
+    if unknown:
+        raise InvalidCandidate(f"edit touches files outside the harness surface: {unknown}")
+    if not any(files[n] != base_files[n] for n in files):
+        raise InvalidCandidate("edit changes nothing")
+    if dest.exists():
+        shutil.rmtree(dest)
+    shutil.copytree(base, dest)
+    for name, text in files.items():
+        (dest / name).write_text(text)
+
+
+def validate(base: Path, candidate: Path) -> None:
+    """Raise InvalidCandidate unless `candidate` is loadable and in bounds."""
+    b, c = harness_files(base), harness_files(candidate)
+    if set(b) != set(c):
+        raise InvalidCandidate(f"file set changed: {sorted(set(b) ^ set(c))}")
+    try:
+        b_set = json.loads(b["settings.json"]) if "settings.json" in b else {}
+        c_set = json.loads(c["settings.json"]) if "settings.json" in c else {}
+        b_tools = json.loads(b["tool_descriptions.json"]) if "tool_descriptions.json" in b else {}
+        c_tools = json.loads(c["tool_descriptions.json"]) if "tool_descriptions.json" in c else {}
+    except json.JSONDecodeError as exc:
+        raise InvalidCandidate(f"invalid JSON: {exc}") from exc
+    if set(c_set) != set(b_set):
+        raise InvalidCandidate(f"settings keys changed: {sorted(set(b_set) ^ set(c_set))}")
+    for key, value in c_set.items():
+        if key.startswith("_"):
+            continue
+        if type(value) is not type(b_set[key]):
+            raise InvalidCandidate(f"setting {key} changed type")
+        from harness_evolve import profiles
+        lo_hi = profiles.active().setting_bounds.get(key)
+        if isinstance(lo_hi, frozenset):
+            if value not in lo_hi:
+                raise InvalidCandidate(f"setting {key}={value!r} not one of {sorted(lo_hi)}")
+        elif lo_hi and not lo_hi[0] <= value <= lo_hi[1]:
+            raise InvalidCandidate(f"setting {key}={value} outside {lo_hi}")
+    if set(c_tools) != set(b_tools):
+        raise InvalidCandidate(f"tool set changed: {sorted(set(b_tools) ^ set(c_tools))}")
+    if any(not str(v).strip() for v in c_tools.values()):
+        raise InvalidCandidate("a tool description is empty")
+    for name in b:
+        if not name.endswith(".prompt"):
+            continue
+        try:
+            extra = _placeholders(c[name]) - _placeholders(b[name])
+        except ValueError as exc:       # unbalanced braces
+            raise InvalidCandidate(f"{name}: malformed template: {exc}") from exc
+        if extra:
+            raise InvalidCandidate(f"{name}: uses placeholders the code never fills: {sorted(extra)}")
+
+
+def diff(base: Path, candidate: Path) -> str:
+    b, c = harness_files(base), harness_files(candidate)
+    out = []
+    for name in sorted(b):
+        if b[name] != c.get(name):
+            out.extend(difflib.unified_diff(b[name].splitlines(keepends=True),
+                                            c.get(name, "").splitlines(keepends=True),
+                                            f"a/{name}", f"b/{name}"))
+    return "".join(out)
+
+
+def changed_files(base: Path, candidate: Path) -> list[str]:
+    b, c = harness_files(base), harness_files(candidate)
+    return sorted(n for n in b if b[n] != c.get(n))