gameability 0.2.0__py3-none-any.whl

gameability/harden.py

@@ -0,0 +1,206 @@
+"""FIND, THEN FIX. Turn the audit into a repair loop.
+
+THE INSIGHT
+The census already computes the repair and discards it. A mutant is certified
+"genuinely broken" by finding a hidden input where it and the reference
+disagree. That input, paired with the reference's output, IS the assertion that
+would have killed it. Emitting it instead of dropping it converts a report into
+a patch.
+
+So hardening is nearly free: audit, collect the witnesses for every mutant that
+survived, append them as new asserts, repeat. Each round the suite gets
+strictly stronger, and it stops when survival stops falling.
+
+THE CONTROL THAT MAKES IT HONEST, AND WITHOUT WHICH THE NUMBER IS A LIE
+Hardening against a set of mutants and then measuring on THAT SAME SET drives
+survival to zero by construction -- we literally wrote the tests that kill
+them. That is not a result, it is a tautology.
+
+So mutants are split. The TRAIN half generates the patches. The TEST half is
+never seen during hardening and is what gets reported. The question that
+actually matters is whether tests written to kill one sample of adversaries
+also kill unseen ones:
+
+    test-set survival falls    -> hardening GENERALISES; the loop makes a
+                                  benchmark genuinely stronger and the product
+                                  is real
+    only train-set survival
+    falls                      -> we are memorising specific mutants; worthless
+
+This is the same held-out logic as heldout_primitives, one level up: there we
+withheld primitives from training, here we withhold adversaries from hardening.
+
+Correctness of the patches is free: every emitted assert is the REFERENCE's own
+behaviour on that input, so a hardened suite can never reject a correct
+solution. The suite only ever gets more specific about what it already meant.
+"""
+import copy
+
+from . import engine, mutate, oracle
+
+
+def witnesses(problem, n_mutants=24, n_hidden=8, seed=0, k_sites=1, timeout=40):
+    """Audit one problem and return the asserts that would kill its survivors.
+
+    Returns (n_broken, n_survived, [assert strings]).
+    """
+    res = engine.audit(problem, n_mutants=n_mutants, n_hidden=n_hidden,
+                       seed=seed, k_sites=k_sites, timeout=timeout,
+                       want_witness=True, mutant_offset=0, mutant_stride=2)
+    if res["status"] != "ok":
+        return 0, 0, []
+    out, seen = [], set()
+    for w in res.get("witness", []):
+        # w = (call_args_string, reference_output_repr)
+        a = "assert %s(%s) == %s" % (problem.entry, w[0], w[1])
+        if a not in seen:
+            seen.add(a)
+            out.append(a)
+    return res["n_broken"], res["n_survived"], out
+
+
+def perturbation_tests(problem, k, seed=0):
+    """`k` fresh asserts, each the REFERENCE's own output on a perturbed input.
+
+    Unlimited supply and correct by construction. The reference is executed once
+    per input; any input on which it raises is skipped rather than guessed at,
+    so no assert here can ever reject a correct solution.
+    """
+    if k <= 0:
+        return []
+    _, calls = oracle.extract_calls(problem.tests, problem.call_alias)
+    fresh = oracle.fresh_calls(calls, n=k * 3, seed=seed)
+    if not fresh:
+        return []
+    recs = engine._run({"ref": problem.reference, "mutants": [],
+                        "setup": problem.setup, "tests": problem.tests,
+                        "fn": problem.entry, "fresh": fresh}, 60)
+    ref = recs.get("ref")
+    if not ref or not ref.get("loaded"):
+        return []
+    out, seen = [], set(problem.tests)
+    for args, res in zip(fresh, ref["hidden"]):
+        if len(out) >= k:
+            break
+        if res[0] != "ok":
+            continue                      # reference raised -- skip, never guess
+        a = "assert %s(%s) == %s" % (problem.entry, args, res[1])
+        if a not in seen:
+            seen.add(a)
+            out.append(a)
+    return out
+
+
+def harden(problem, rounds=3, n_mutants=24, seed=0, verbose=False,
+           target_tests=0):
+    """Iteratively strengthen a problem's test suite.
+
+    TRAIN mutants (seed) generate the patches; TEST mutants (seed+500) are held
+    out and are the only thing reported. Returns a dict of the trajectory.
+    """
+    cur = copy.copy(problem)
+    cur.tests = list(problem.tests)
+
+    def measure(p, offset):
+        # offset 0 = TRAIN, the mutants patches were written against.
+        # offset 1 = TEST, never seen during hardening. Only TEST is reported.
+        r = engine.audit(p, n_mutants=n_mutants, seed=seed,
+                         mutant_offset=offset, mutant_stride=2)
+        if r["status"] != "ok" or r["n_broken"] == 0:
+            return None
+        return r["n_survived"] / r["n_broken"]
+
+    hist = {"pid": problem.pid, "train": [], "test": [], "n_tests": [],
+            "added": []}
+    for rd in range(rounds + 1):
+        tr = measure(cur, 0)
+        te = measure(cur, 1)                   # NEVER used to generate patches
+        hist["train"].append(tr)
+        hist["test"].append(te)
+        hist["n_tests"].append(len(cur.tests))
+        if verbose:
+            print("    round %d  tests=%-3d train=%s test=%s"
+                  % (rd, len(cur.tests),
+                     "n/a" if tr is None else "%.3f" % tr,
+                     "n/a" if te is None else "%.3f" % te))
+        if rd == rounds:
+            break
+        _, surv, new = witnesses(cur, n_mutants=n_mutants, seed=seed)
+        new = [a for a in new if a not in cur.tests]
+
+        # TOP UP with perturbation tests. Witnesses run out -- they are bounded
+        # by the mutant sample -- and when they do the loop used to halt with
+        # the held-out half still leaking. Perturbation inputs are unlimited and
+        # the floor-fix curve shows they carry survival from 0.060 to 0.004 on
+        # their own. Targeted first, then bulk.
+        if target_tests and len(cur.tests) + len(new) < target_tests:
+            need = target_tests - len(cur.tests) - len(new)
+            new += perturbation_tests(cur, need, seed=seed + 100 * rd)
+
+        hist["added"].append(len(new))
+        if not new:
+            # neither source produced anything: witnesses exhausted AND the
+            # suite is already at its target size
+            break
+        cur.tests = cur.tests + new
+    hist["final_tests"] = list(cur.tests)
+    hist["hardened"] = cur
+    return hist
+
+
+def _carry(seq, depth):
+    """Extend a trajectory to `depth` by repeating its last value.
+
+    A problem that ran out of survivors to patch STOPPED because it converged.
+    It is at that value for every later round, not missing from them. Averaging
+    column-wise without this computes each round over only the problems still
+    running -- which are exactly the ones with more left to fix -- and makes
+    hardening look better the longer it goes.
+    """
+    if not seq:
+        return [None] * depth
+    out = list(seq)
+    while len(out) < depth:
+        out.append(out[-1])
+    return out
+
+
+def summarise_harden(rows):
+    """Aggregate trajectories; the TEST columns are the result.
+
+    Every column averages the SAME population, by carrying converged problems
+    forward rather than dropping them.
+    """
+    n = len(rows)
+    depth = max((len(r["test"]) for r in rows), default=0)
+    out = {"problems": n, "rounds": depth, "train": [], "test": [],
+           "tests": [], "n_scored": [], "dropped_unmatched": 0}
+    # MATCHED POPULATION. A problem with no genuinely-broken mutant at round 0
+    # scores None there, and can acquire scorable mutants later once the suite
+    # grows -- so the population drifts (53 -> 55 in one run) and the columns
+    # stop being comparable. Restrict to problems scorable at EVERY round.
+    def _full(r):
+        c = _carry(r["test"], depth)
+        return all(v is not None for v in c)
+
+    matched = [r for r in rows if _full(r)]
+    dropped = len(rows) - len(matched)
+    rows = matched
+
+    tr_all = [_carry(r["train"], depth) for r in rows]
+    te_all = [_carry(r["test"], depth) for r in rows]
+    nt_all = [_carry(r["n_tests"], depth) for r in rows]
+    for i in range(depth):
+        tr = [c[i] for c in tr_all if c[i] is not None]
+        te = [c[i] for c in te_all if c[i] is not None]
+        nt = [c[i] for c in nt_all if c[i] is not None]
+        out["train"].append(sum(tr) / len(tr) if tr else None)
+        out["test"].append(sum(te) / len(te) if te else None)
+        out["tests"].append(sum(nt) / len(nt) if nt else None)
+        out["n_scored"].append(len(te))
+    # a column scored over fewer problems than another is not comparable
+    scored = set(out["n_scored"])
+    out["population_stable"] = len(scored) <= 1
+    out["dropped_unmatched"] = dropped
+    out["added_total"] = sum(sum(r["added"]) for r in rows)
+    return out

gameability/loaders.py

@@ -0,0 +1,290 @@
+"""Turn a benchmark into a list of Problems.
+
+Three sources:
+  * jsonl      -- your own benchmark, one problem per line (format in the README)
+  * mbpp       -- Google's MBPP, "full" config, via Hugging Face `datasets`
+  * humaneval  -- OpenAI's HumanEval, via Hugging Face `datasets`
+
+Every loader returns (problems, dropped) so the caller can say how many records
+never reached the audit instead of silently shrinking the denominator.
+"""
+import ast
+import json
+
+from .engine import Problem
+
+
+def _defined_functions(src):
+    try:
+        tree = ast.parse(src)
+    except SyntaxError:
+        return []
+    return [n.name for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)]
+
+
+def _called_names(tests):
+    out = []
+    for t in tests:
+        try:
+            tree = ast.parse(t.strip())
+        except SyntaxError:
+            continue
+        out += [n.func.id for n in ast.walk(tree)
+                if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)]
+    return out
+
+
+def _entry(reference, tests):
+    """The function the reference DEFINES and the tests CALL.
+
+    Taking the most-called name instead would pick `set` or `len` out of an
+    assert like `assert set(f(x)) == set(y)`, and the audit would then measure
+    nothing at all while still reporting a number.
+    """
+    defined, called = _defined_functions(reference), _called_names(tests)
+    return next((c for c in called if c in defined), None)
+
+
+def from_jsonl(path):
+    problems, dropped = [], 0
+    with open(path, encoding="utf-8") as f:
+        for n, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            r = json.loads(line)
+            for key in ("reference", "tests"):
+                if key not in r:
+                    raise SystemExit("%s line %d: missing required field %r" % (path, n, key))
+            tests = r["tests"] if isinstance(r["tests"], list) else [r["tests"]]
+            entry = r.get("entry") or _entry(r["reference"], tests)
+            if not entry:
+                dropped += 1
+                continue
+            problems.append(Problem(pid=str(r.get("id", n)), reference=r["reference"], tests=tests,
+                                    setup=r.get("setup", ""), entry=entry,
+                                    hidden=list(r.get("hidden", [])),
+                                    call_alias=r.get("call_alias")))
+    return problems, dropped
+
+
+def _load_dataset(*args):
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        raise SystemExit("this loader needs Hugging Face datasets: pip install 'gameability[hf]'")
+    return load_dataset(*args)
+
+
+def mbpp():
+    """MBPP 'full': `code` is the reference, `test_list` the visible suite, and
+    `challenge_test_list` -- where it exists -- a held-out suite the benchmark
+    already has, credited as extra hidden inputs rather than ignored."""
+    ds = _load_dataset("google-research-datasets/mbpp", "full")
+    # The 'prompt' split is the 10 few-shot examples, not the benchmark.
+    rows = [r for name, split in ds.items() if name != "prompt" for r in split]
+    problems, dropped = [], 0
+    for r in rows:
+        tests = list(r.get("test_list") or [])
+        entry = _entry(r["code"], tests) if tests and r.get("code") else None
+        if entry is None:
+            dropped += 1
+            continue
+        problems.append(Problem(pid=str(r["task_id"]), reference=r["code"], tests=tests,
+                                setup=r.get("test_setup_code") or "", entry=entry,
+                                hidden=list(r.get("challenge_test_list") or [])))
+    return problems, dropped
+
+
+def humaneval():
+    """HumanEval's `test` defines check(candidate); calling it is the suite."""
+    rows = _load_dataset("openai/openai_humaneval")["test"]
+    problems = []
+    for r in rows:
+        problems.append(Problem(pid=r["task_id"], reference=r["prompt"] + r["canonical_solution"],
+                                tests=[r["test"] + "\ncheck(%s)\n" % r["entry_point"]],
+                                entry=r["entry_point"], call_alias="candidate"))
+    return problems, 0
+
+
+def _split_evalplus(test_src):
+    """EvalPlus runs all of a problem's inputs in one loop:
+
+        inputs = [...]; results = [...]
+        for i, (inp, exp) in enumerate(zip(inputs, results)):
+            assertion(fn(*inp), exp, ATOL)
+
+    Run as one test, a single step budget would cover hundreds of inputs and cut
+    the reference off. So the loop is unrolled into one assert per input, each
+    built from the exact source text of that input and its expected value.
+    Returns (prelude, [(args_src, exp_src)], atol_src, called_name) or None.
+    """
+    try:
+        tree = ast.parse(test_src)
+    except SyntaxError:
+        return None
+    # ast.get_source_segment re-splits the whole source on every call, which is
+    # quadratic on EvalPlus files with thousands of inputs. AST columns are UTF-8
+    # byte offsets, so slice the encoded source with line offsets computed once.
+    raw = test_src.encode("utf-8")
+    starts, pos = [0], 0
+    for ln in raw.splitlines(keepends=True):
+        pos += len(ln)
+        starts.append(pos)
+
+    def seg(node):
+        a = starts[node.lineno - 1] + node.col_offset
+        b = starts[node.end_lineno - 1] + node.end_col_offset
+        return raw[a:b].decode("utf-8")
+
+    lit, loop = {}, None
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) \
+                and node.targets[0].id in ("inputs", "results"):
+            lit[node.targets[0].id] = node.value
+        if isinstance(node, ast.For) and loop is None:
+            loop = node
+    if set(lit) != {"inputs", "results"} or loop is None:
+        return None
+    if not isinstance(lit["inputs"], ast.List) or not isinstance(lit["results"], ast.List):
+        return None
+    call = next((n for n in ast.walk(loop) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
+                 and n.func.id == "assertion"), None)
+    if call is None or len(call.args) != 3 or not isinstance(call.args[0], ast.Call) \
+            or not isinstance(call.args[0].func, ast.Name):
+        return None
+    called = call.args[0].func.id
+    atol = seg(call.args[2])
+    if len(lit["inputs"].elts) != len(lit["results"].elts):
+        return None
+    pairs = []
+    for inp, exp in zip(lit["inputs"].elts, lit["results"].elts):
+        if not isinstance(inp, (ast.List, ast.Tuple)):
+            return None
+        args = ", ".join(seg(a) for a in inp.elts)
+        pairs.append((args, seg(exp)))
+    keep = [n for n in tree.body if isinstance(n, (ast.Import, ast.ImportFrom))
+            or (isinstance(n, ast.FunctionDef) and n.name != "check")]
+    prelude = "\n\n".join(seg(n) for n in keep)
+    return prelude, pairs, atol, called
+
+
+def mbpp_plus(original_reference=False):
+    """EvalPlus MBPP+: the visible suite is EvalPlus's full input set; fresh inputs
+    are harvested from the ORIGINAL MBPP test_list, so MBPP and MBPP+ face the
+    same adversary inputs. EvalPlus also rewrote many references, so with
+    original_reference=True the ORIGINAL MBPP solution is used instead and only
+    the tests differ from plain MBPP."""
+    ds = _load_dataset("evalplus/mbppplus")["test"]
+    orig = None
+    if original_reference:
+        full = _load_dataset("google-research-datasets/mbpp", "full")
+        orig = {str(r["task_id"]): r["code"] for n, s in full.items() if n != "prompt" for r in s}
+    problems, dropped = [], 0
+    for r in ds:
+        if orig is not None:
+            if str(r["task_id"]) not in orig:
+                dropped += 1
+                continue
+            r = dict(r, code=orig[str(r["task_id"])])
+        base = list(r.get("test_list") or [])
+        entry = _entry(r["code"], base) if base else None
+        sp = _split_evalplus(r["test"]) if entry else None
+        if sp is None or sp[3] != entry:
+            dropped += 1
+            continue
+        prelude, pairs, atol, _ = sp
+        setup = "\n".join(list(r.get("test_imports") or []) + [prelude])
+        tests = ["assertion(%s(%s), %s, %s)" % (entry, a, e, atol) for a, e in pairs]
+        problems.append(Problem(pid=str(r["task_id"]), reference=r["code"], tests=tests, setup=setup,
+                                entry=entry, harvest=base))
+    return problems, dropped
+
+
+def humaneval_plus(original_reference=False):
+    """EvalPlus HumanEval+: visible suite = the full EvalPlus input set; fresh inputs
+    are harvested from the ORIGINAL HumanEval test of the same task. With
+    original_reference=True the original HumanEval solution is used as well."""
+    ds = _load_dataset("evalplus/humanevalplus")["test"]
+    he = {r["task_id"]: r for r in _load_dataset("openai/openai_humaneval")["test"]}
+    orig = {t: r["test"] for t, r in he.items()}
+    problems, dropped = [], 0
+    for r in ds:
+        sp = _split_evalplus(r["test"])
+        if sp is None or r["task_id"] not in orig:
+            dropped += 1
+            continue
+        prelude, pairs, atol, called = sp
+        entry = r["entry_point"]
+        tests = ["assertion(%s(%s), %s, %s)" % (entry, a, e, atol) for a, e in pairs]
+        src = he[r["task_id"]] if original_reference else r
+        problems.append(Problem(pid=r["task_id"], reference=src["prompt"] + src["canonical_solution"], tests=tests,
+                                setup=prelude, entry=entry, call_alias="candidate",
+                                harvest=[orig[r["task_id"]]]))
+    return problems, dropped
+
+
+def mbpp_sanitized():
+    ds = _load_dataset("google-research-datasets/mbpp", "sanitized")
+    problems, dropped = [], 0
+    for name, split in ds.items():
+        if name == "prompt":
+            continue
+        for r in split:
+            tests = list(r.get("test_list") or [])
+            entry = _entry(r["code"], tests) if tests and r.get("code") else None
+            if entry is None:
+                dropped += 1
+                continue
+            problems.append(Problem(pid=str(r["task_id"]), reference=r["code"], tests=tests,
+                                    setup="\n".join(r.get("test_imports") or []), entry=entry))
+    return problems, dropped
+
+
+def kodcode(path):
+    """KodCode rows saved as jsonl (fields: question_id, solution, test, test_info).
+
+    The test is a pytest module that imports the solution (`from solution import f`).
+    The import is removed (the solution is already in the namespace) and every plain
+    `test_*()` function -- no arguments, no decorators -- becomes its own test, so each
+    gets its own step budget. Fixture and parametrised tests cannot be called without
+    pytest's runner and are skipped; a problem with none left is dropped and counted.
+    """
+    import re
+    problems, dropped = [], 0
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            if not line.strip():
+                continue
+            r = json.loads(line)
+            info = r.get("test_info") or []
+            entry = info[0].get("function_name") if info else None
+            try:
+                top = {n.name for n in ast.parse(r["solution"]).body if isinstance(n, ast.FunctionDef)}
+            except SyntaxError:
+                top = set()
+            if not entry or entry not in top:
+                dropped += 1
+                continue
+            src = re.sub(r"(?m)^\s*(from\s+solution\s+import\s+.*|import\s+solution\s*)$", "", r["test"])
+            src = re.sub(r"\bsolution\.", "", src)
+            try:
+                tree = ast.parse(src)
+            except SyntaxError:
+                dropped += 1
+                continue
+            names = [n.name for n in tree.body if isinstance(n, ast.FunctionDef) and n.name.startswith("test")
+                     and not n.decorator_list and not n.args.args]
+            if not names:
+                dropped += 1
+                continue
+            tests = ["%s\n\n%s()\n" % (src, n) for n in names]
+            problems.append(Problem(pid=str(r.get("question_id", r.get("_row_idx"))), reference=r["solution"],
+                                    tests=tests, entry=entry, harvest=[src]))
+    return problems, dropped
+
+
+SOURCES = {"mbpp": mbpp, "humaneval": humaneval, "mbpp-sanitized": mbpp_sanitized,
+           "mbpp-plus": mbpp_plus, "humaneval-plus": humaneval_plus,
+           "mbpp-plus-origref": lambda: mbpp_plus(original_reference=True),
+           "humaneval-plus-origref": lambda: humaneval_plus(original_reference=True)}

gameability/mutate.py

@@ -0,0 +1,155 @@
+"""Single-site semantic mutation of a reference solution.
+
+WHY THIS EXISTS
+A benchmark claims to measure a capability. The claim is only as strong as its
+ability to REJECT a solution that lacks the capability. So we manufacture such
+solutions on purpose: take the reference, break it at exactly one site, and ask
+whether the benchmark notices.
+
+This is not bug-finding. The reference is assumed correct. The mutant is the
+adversary, and the benchmark's test suite is the thing under test.
+
+Each mutant differs from the reference at ONE site, so a surviving mutant names
+a specific behaviour the suite does not constrain.
+"""
+import ast
+import copy
+
+CMP_SWAP = {ast.Lt: ast.LtE, ast.LtE: ast.Lt, ast.Gt: ast.GtE, ast.GtE: ast.Gt,
+            ast.Eq: ast.NotEq, ast.NotEq: ast.Eq, ast.In: ast.NotIn,
+            ast.NotIn: ast.In}
+BIN_SWAP = {ast.Add: ast.Sub, ast.Sub: ast.Add, ast.Mult: ast.FloorDiv,
+            ast.FloorDiv: ast.Mult, ast.Div: ast.Mult, ast.Mod: ast.Mult,
+            ast.Pow: ast.Mult}
+BOOL_SWAP = {ast.And: ast.Or, ast.Or: ast.And}
+
+
+def sites(src):
+    """Every position where a single-site mutation is defined.
+
+    Indices are positions in ast.walk order, which is deterministic for a given
+    source string, so a site index is a stable name for a mutation.
+    """
+    try:
+        tree = ast.parse(src)
+    except SyntaxError:
+        return []
+    out = []
+    for i, n in enumerate(ast.walk(tree)):
+        if isinstance(n, ast.Compare) and len(n.ops) == 1 and type(n.ops[0]) in CMP_SWAP:
+            out.append((i, "cmp"))
+        elif isinstance(n, ast.BinOp) and type(n.op) in BIN_SWAP:
+            out.append((i, "arith"))
+        elif isinstance(n, ast.BoolOp) and type(n.op) in BOOL_SWAP:
+            out.append((i, "bool"))
+        elif isinstance(n, ast.Constant) and isinstance(n.value, bool):
+            out.append((i, "boolconst"))
+        elif isinstance(n, ast.Constant) and isinstance(n.value, int):
+            # bool is a subclass of int, so the branch above must come first
+            out.append((i, "intconst"))
+        elif isinstance(n, ast.Return) and n.value is not None:
+            out.append((i, "return_none"))
+    return out
+
+
+def mutate(src, index, kind):
+    """Apply one mutation and return the new source, or None if inapplicable."""
+    tree = ast.parse(src)
+    nodes = list(ast.walk(tree))
+    if index >= len(nodes):
+        return None
+    n = nodes[index]
+    try:
+        if kind == "cmp":
+            n.ops = [CMP_SWAP[type(n.ops[0])]()]
+        elif kind == "arith":
+            n.op = BIN_SWAP[type(n.op)]()
+        elif kind == "bool":
+            n.op = BOOL_SWAP[type(n.op)]()
+        elif kind == "boolconst":
+            n.value = not n.value
+        elif kind == "intconst":
+            n.value = n.value + 1
+        elif kind == "return_none":
+            n.value = ast.Constant(value=None)
+        else:
+            return None
+    except (KeyError, AttributeError, IndexError):
+        return None
+    try:
+        return ast.unparse(ast.fix_missing_locations(tree))
+    except Exception:
+        return None
+
+
+def generate_multi(src, limit=24, k=2, seed=0):
+    """Mutants that break the reference at `k` sites at once.
+
+    Single-site mutation is the WEAKEST possible adversary, so a survival rate
+    measured with it is a floor and not an estimate. A real policy optimising
+    against a test suite explores an incomparably larger space, and the honest
+    way to move toward that is to let the adversary combine edits.
+
+    A k-site mutant is not automatically harder to catch: two errors can cancel,
+    which is why the hidden-input oracle still has to certify every one of them
+    as genuinely broken. What changes is the size of the space being searched.
+    """
+    import random as _r
+    rng = _r.Random(seed)
+    all_sites = sites(src)
+    if len(all_sites) < k:
+        return []
+    out, seen = [], set()
+    for _ in range(limit * 12):
+        if len(out) >= limit:
+            break
+        picked = rng.sample(all_sites, k)
+        key = tuple(sorted(i for i, _ in picked))
+        if key in seen:
+            continue
+        seen.add(key)
+        cur = src
+        ok = True
+        for idx, kind in picked:
+            # Re-derive the site list after each edit: ast.walk indices shift
+            # when a node is replaced, and reusing stale indices would silently
+            # mutate the wrong node -- producing a mutant nobody designed.
+            live = dict(sites(cur))
+            if idx not in live or live[idx] != kind:
+                ok = False
+                break
+            nxt = mutate(cur, idx, kind)
+            if nxt is None:
+                ok = False
+                break
+            cur = nxt
+        if ok and cur.strip() != src.strip():
+            out.append({"kind": "multi%d" % k, "site": key, "src": cur})
+    return out
+
+
+def generate(src, limit=24):
+    """All single-site mutants of `src`, capped at `limit`, deterministically.
+
+    The cap is spread ACROSS operator kinds rather than taking the first N
+    sites, because the first N sites of a long function are all in its opening
+    lines and would test only one region of the behaviour.
+    """
+    by_kind = {}
+    for i, kind in sites(src):
+        by_kind.setdefault(kind, []).append(i)
+    out, exhausted = [], False
+    round_no = 0
+    while len(out) < limit and not exhausted:
+        exhausted = True
+        for kind in sorted(by_kind):
+            idxs = by_kind[kind]
+            if round_no < len(idxs):
+                exhausted = False
+                m = mutate(src, idxs[round_no], kind)
+                if m is not None and m.strip() != src.strip():
+                    out.append({"kind": kind, "site": idxs[round_no], "src": m})
+                if len(out) >= limit:
+                    break
+        round_no += 1
+    return out