gameability 0.2.0__py3-none-any.whl

gameability/oracle.py

@@ -0,0 +1,232 @@
+"""Hidden inputs, derived from the benchmark's own visible tests.
+
+WHY THIS EXISTS
+A mutant that survives the visible tests proves nothing on its own -- it might
+be an EQUIVALENT mutant, semantically identical to the reference, in which case
+passing is correct and the test suite is blameless. Counting those as gameable
+would be the single easiest way to manufacture an impressive false result.
+
+So every surviving mutant must first be shown genuinely broken, and that needs
+inputs the visible tests never show. We build them by perturbing the literal
+arguments already present in the benchmark's asserts: the types stay right by
+construction, so no type inference is needed, and the values are fresh.
+
+This is the same construction that took a lookup table over our own generated
+problems from 60/60 to 0/60.
+"""
+import ast
+import random
+import string
+
+
+def extract_calls(tests, fn=None):
+    """Pull `fn(literal, ...)` argument tuples out of a list of assert strings."""
+    calls, name = [], fn
+    for t in tests:
+        try:
+            tree = ast.parse(t.strip())
+        except SyntaxError:
+            continue
+        for n in ast.walk(tree):
+            if not isinstance(n, ast.Call) or not isinstance(n.func, ast.Name):
+                continue
+            if name is None:
+                name = n.func.id
+            if n.func.id != name or n.keywords:
+                continue
+            try:
+                args = [ast.literal_eval(a) for a in n.args]
+            except (ValueError, SyntaxError, TypeError):
+                continue
+            calls.append(args)
+    return name, calls
+
+
+def entry_name(tests):
+    """The function the asserts call most often -- the problem's entry point."""
+    counts = {}
+    for t in tests:
+        try:
+            tree = ast.parse(t.strip())
+        except SyntaxError:
+            continue
+        for n in ast.walk(tree):
+            if isinstance(n, ast.Call) and isinstance(n.func, ast.Name):
+                counts[n.func.id] = counts.get(n.func.id, 0) + 1
+    if not counts:
+        return None
+    return max(sorted(counts), key=lambda k: counts[k])
+
+
+def perturb(v, rng, depth=0):
+    """Return a fresh value of the same shape and type as `v`."""
+    if depth > 4:
+        return v
+    if isinstance(v, bool):
+        return not v
+    if isinstance(v, int):
+        d = rng.choice([-3, -2, -1, 1, 2, 3, 5, 7])
+        return v + d
+    if isinstance(v, float):
+        return round(v + rng.uniform(-3, 3), 4)
+    if isinstance(v, str):
+        if not v:
+            return rng.choice(["a", "ab", "xyz"])
+        pool = string.ascii_lowercase
+        if v.isupper():
+            pool = string.ascii_uppercase
+        elif v.isdigit():
+            pool = string.digits
+        n = max(1, len(v) + rng.choice([-1, 0, 0, 1]))
+        return "".join(rng.choice(pool) for _ in range(n))
+    if isinstance(v, (list, tuple, set)):
+        items = list(v)
+        if not items:
+            return type(v)()
+        n = max(1, len(items) + rng.choice([-1, 0, 0, 1]))
+        base = [items[i % len(items)] for i in range(n)]
+        out = [perturb(x, rng, depth + 1) for x in base]
+        return type(v)(out) if not isinstance(v, set) else set(out)
+    if isinstance(v, dict):
+        return {k: perturb(val, rng, depth + 1) for k, val in v.items()}
+    return v
+
+
+# ---- CONTRACT-PRESERVING INPUTS (v0.2, contract_inputs_v1)
+# The free generator above changes values with no knowledge of a problem's rules, so it can ask a
+# binary search about an unsorted list or a parenthesis parser about random letters. On such inputs the
+# reference's answer is arbitrary, and a mutant that differs there was counted as broken: measured on the
+# problems EvalPlus covers, 14% of flagged MBPP and 21% of flagged HumanEval problems were flagged only by
+# such inputs (census_contract_check_v1). This generator learns each argument's observed domain from ALL
+# of a problem's visible calls and stays inside it.
+
+def _is_sorted(x):
+    try:
+        return all(x[i] <= x[i + 1] for i in range(len(x) - 1))
+    except TypeError:
+        return False
+
+
+def profile(values):
+    """What every observed value at one argument position has in common."""
+    vals = [v for v in values]
+    if not vals:
+        return {"kind": "any"}
+    if all(isinstance(v, bool) for v in vals):
+        return {"kind": "bool"}
+    if all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in vals):
+        lo = min(vals)
+        return {"kind": "num", "lb": 1 if lo >= 1 else (0 if lo >= 0 else None)}
+    if all(isinstance(v, str) for v in vals):
+        spaced = any(" " in v.strip() for v in vals)
+        tokens = sorted({t for v in vals for t in v.split()}) if spaced else []
+        chars = sorted({c for v in vals for c in v})
+        return {"kind": "str", "chars": chars, "tokens": tokens}
+    if all(isinstance(v, (list, tuple)) for v in vals):
+        evid = [v for v in vals if len(v) >= 2]
+        flat = [x for v in vals for x in v]
+        rect = bool(vals) and all(len(v) >= 1 and all(isinstance(r, (list, tuple)) for r in v) and
+                                  len({len(r) for r in v}) == 1 for v in vals)
+        square = rect and all(len(v) == len(v[0]) for v in vals)
+        def perm1n(v):
+            f = [x for r in v for x in r] if rect else list(v)
+            return all(isinstance(x, int) and not isinstance(x, bool) for x in f) and sorted(f) == list(range(1, len(f) + 1))
+        return {"kind": "seq",
+                "sorted": bool(evid) and not rect and all(_is_sorted(v) for v in evid),
+                "unique": bool(evid) and not rect and all(len(set(map(repr, v))) == len(v) for v in evid),
+                "rect": rect, "square": square,
+                "perm1n": bool(evid) and all(perm1n(v) for v in vals),
+                "elem": profile([x for r in flat for x in r] if rect else flat)}
+    return {"kind": "any"}
+
+
+def perturb_contract(v, rng, prof, depth=0):
+    """A fresh value of the same shape and type as `v`, inside the domain `prof` was learned from."""
+    k = prof.get("kind")
+    if depth > 4 or k == "any":
+        return perturb(v, rng, depth)
+    if k == "bool" and isinstance(v, bool):
+        return not v
+    if k == "num" and isinstance(v, (int, float)) and not isinstance(v, bool):
+        out = v + rng.choice([-3, -2, -1, 1, 2, 3, 5, 7]) if isinstance(v, int) else round(v + rng.uniform(-3, 3), 4)
+        lb = prof.get("lb")
+        if lb is not None and out < lb:
+            out = lb + (lb - out)                         # reflect back inside the observed domain
+        return out
+    if k == "str" and isinstance(v, str):
+        if prof["tokens"]:                                # whitespace-separated tokens: recombine observed tokens
+            n = max(1, len(v.split()) + rng.choice([-1, 0, 0, 1]))
+            return " ".join(rng.choice(prof["tokens"]) for _ in range(n))
+        pool = prof["chars"] or list(string.ascii_lowercase)
+        n = max(1, len(v) + rng.choice([-1, 0, 0, 1])) if v else 1
+        return "".join(rng.choice(pool) for _ in range(n))
+    if k == "seq" and isinstance(v, (list, tuple)):
+        if not v:
+            return type(v)()
+        if prof["rect"]:
+            rows, cols = len(v), len(v[0])
+            if prof["square"]:
+                rows = cols = max(1, rows + rng.choice([-1, 0, 0, 1]))
+            else:
+                rows = max(1, rows + rng.choice([-1, 0, 0, 1]))
+            if prof["perm1n"]:
+                vals = list(range(1, rows * cols + 1)); rng.shuffle(vals)
+            else:
+                src = [x for r in v for x in r]
+                vals = [perturb_contract(src[i % len(src)], rng, prof["elem"], depth + 1) for i in range(rows * cols)]
+            out = [vals[i * cols:(i + 1) * cols] for i in range(rows)]
+            return type(v)(type(v[0])(r) for r in out)
+        n = max(1, len(v) + rng.choice([-1, 0, 0, 1]))
+        if prof["perm1n"]:
+            out = list(range(1, n + 1)); rng.shuffle(out)
+        else:
+            base = [v[i % len(v)] for i in range(n)]
+            out = [perturb_contract(x, rng, prof["elem"], depth + 1) for x in base]
+            if prof["unique"]:
+                seen, uniq = set(), []
+                for x in out:
+                    tries = 0
+                    while repr(x) in seen and tries < 20:
+                        x = perturb_contract(x, rng, prof["elem"], depth + 1); tries += 1
+                    if repr(x) not in seen:
+                        seen.add(repr(x)); uniq.append(x)
+                out = uniq or out[:1]
+            if prof["sorted"]:
+                try:
+                    out = sorted(out)
+                except TypeError:
+                    pass
+        return type(v)(out)
+    return perturb(v, rng, depth)
+
+
+def fresh_calls(calls, n=8, seed=0, inputs="contract"):
+    """Fresh argument tuples, as source text, ready to be executed.
+
+    inputs="contract" (default from v0.2) stays inside each argument's observed domain;
+    inputs="free" is v0.1's generator, kept so its published numbers remain reproducible."""
+    if not calls:
+        return []
+    if inputs not in ("contract", "free"):
+        raise ValueError("inputs must be 'contract' or 'free', got %r" % (inputs,))
+    rng = random.Random(seed)
+    profs = {}
+    if inputs == "contract":
+        for ar in {len(c) for c in calls}:
+            same = [c for c in calls if len(c) == ar]
+            profs[ar] = [profile([c[j] for c in same]) for j in range(ar)]
+    out, seen = [], {repr(c) for c in calls}
+    for i in range(n * 4):
+        if len(out) >= n:
+            break
+        base = calls[i % len(calls)]
+        if inputs == "contract":
+            cand = [perturb_contract(a, rng, p) for a, p in zip(base, profs[len(base)])]
+        else:
+            cand = [perturb(a, rng) for a in base]
+        r = repr(cand)
+        if r in seen:
+            continue
+        seen.add(r)
+        out.append(", ".join(repr(a) for a in cand))
+    return out

gameability-0.2.0.dist-info/METADATA

@@ -0,0 +1,162 @@
+Metadata-Version: 2.4
+Name: gameability
+Version: 0.2.0
+Summary: Find the problems in a code benchmark that accept broken solutions.
+Author: Empyrean Intelligence
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Empyrean-Intelligence/gameability
+Project-URL: Source, https://github.com/Empyrean-Intelligence/gameability
+Project-URL: Issues, https://github.com/Empyrean-Intelligence/gameability/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: hf
+Requires-Dist: datasets; extra == "hf"
+Dynamic: license-file
+
+# gameability
+
+**Find the problems in a code benchmark that accept broken solutions.**
+
+When a model learns from tests, the tests are its reward. A self-improving system trained on tests that
+accept broken code learns to pass weak tests, not to write correct code. `gameability` measures how often
+that can happen. It is part of the verification layer we use at Empyrean Intelligence for self-improvement
+work, released for anyone who builds benchmarks or RL training sets.
+
+A benchmark score means something only if the benchmark rejects solutions that lack the
+skill it measures. `gameability` checks that directly. It takes each problem's reference
+solution, breaks it in one small place, confirms the result really is wrong, and then
+asks whether the problem's own tests notice.
+
+```
+pip install "gameability[hf]"
+gameability check mbpp
+```
+
+On MBPP, **17.9% of problems (169 of 942 judged) accept a broken solution**; on HumanEval,
+**10.3% (16 of 155)**. Each run takes minutes on a laptop, with no model and no API key.
+
+**Confirmed on valid inputs.** On the problems EvalPlus also covers, every accepted broken solution
+was re-run against EvalPlus's independent tests, whose inputs respect each problem's rules:
+**95.2% of flagged MBPP problems and 84.6% of flagged HumanEval problems are confirmed** (the broken
+solution passes the original tests and fails EvalPlus). The two unconfirmed HumanEval problems
+(116 and 129) are ones whose own visible tests contradict their written specification.
+
+**What changed in v0.2.** v0.1 built fresh inputs by changing values freely, which sometimes asked
+a problem about inputs it never promised to handle (an unsorted list for binary search, random
+letters for a parenthesis parser). v0.2 learns each argument's domain from the problem's own
+visible tests and stays inside it: number signs and minimums, the characters or whole tokens a
+string uses, and whether lists are sorted, unique, rectangular or a permutation. On the same
+problems this raised the confirmed share from 85.7% to 95.2% (MBPP) and 78.6% to 84.6% (HumanEval)
+while keeping 93.1% of v0.1's confirmed detections. `--inputs free` reproduces v0.1.
+
+## Scoreboard
+
+Problems that accept at least one broken solution (wrong on at least one fresh input), with 95% intervals. Defaults (24 broken
+versions per problem, 8 fresh inputs, seed 0), laptop CPU, October 2026.
+
+| Benchmark | Weak problems | Command |
+|---|---|---|
+| MBPP | 17.9% (169 of 942) [15.6, 20.5] | `gameability check mbpp` |
+| MBPP sanitized | 13.8% (57 of 413) [10.8, 17.5] | `gameability check mbpp-sanitized` |
+| HumanEval | 10.3% (16 of 155) [6.5, 16.1] | `gameability check humaneval` |
+| KodCode (RL training set, 1,000-row random sample) | 9.7% (45 of 466) [7.3, 12.7] | `gameability check kodcode:rows.jsonl` |
+| HumanEval+ (EvalPlus) | 3.4% (5 of 147) [1.5, 7.7] | `gameability check humaneval-plus` |
+| MBPP+ (EvalPlus) | 2.0% (7 of 353) [1.0, 4.0] | `gameability check mbpp-plus` |
+
+EvalPlus's stronger tests look like a large improvement, but EvalPlus also rewrote most reference
+solutions. Holding the original reference fixed (`mbpp-plus-origref`, `humaneval-plus-origref`), the weak
+share roughly halves (MBPP 8.7% to 3.1% on 254 problems, HumanEval 10.6% to 4.9% on 123), and at these
+sample sizes the intervals overlap. KodCode's judged set is the problems whose tests call the function with
+literal arguments (466 of 801 loadable). `rows.jsonl` is KodCode rows saved as JSON lines; the sample above
+is reproducible with `python tests/sample_kodcode.py rows.jsonl` (positions from `random.Random(0)`).
+
+## What it prints
+
+```
+  415 -- 6 of 11 broken solutions pass every visible test
+      -    if arr_len < 2:
+      +    if arr_len <= 2:
+      a test that catches it: assert max_Product([4, 9]) == (4, 9)
+```
+
+For every weak problem you get the one line that was changed, how many broken versions got
+through, and an assert that would have caught one.
+
+**`--fixes fixes.py` writes those asserts to a file; review them before adding them.** Measured on
+34 MBPP/HumanEval problems against a 1.5B model's real solutions: v0.2's suggested asserts wrongly
+reject **2.6% of its correct solutions** (8 of 307; v0.1: 15.3%). They target the broken versions
+gameability builds, not the mistakes a particular model makes: they caught 4 of that model's 24
+real wrong solutions. Treat them as leads, not as a test suite.
+
+## How it works
+
+For each problem the benchmark supplies (reference solution, visible tests):
+
+1. The reference must pass its own visible tests. If it doesn't, the problem is excluded and counted.
+2. Fresh inputs are built by varying the arguments in the visible tests. The reference's
+   outputs on them are the hidden answer key.
+3. The reference is mutated at a single site: a comparison flipped, an operator swapped, a
+   constant nudged, a `return` replaced with `return None`.
+4. A mutant counts as **broken** only if it gives a different answer from the reference on
+   at least one fresh input. Mutants that behave identically are thrown away, so they can't
+   inflate the number.
+5. A broken mutant that still passes every visible test is a **survivor**. A problem with at
+   least one survivor accepts a wrong solution.
+
+Benchmark code runs in a child process with a step budget, so a runaway mutant is stopped at
+the same point on any machine; a wall-clock limit is only a backstop. Mutants that never
+finish are reported as truncated, never counted as rejected.
+
+## Your own benchmark
+
+One problem per line in a `.jsonl` file:
+
+```json
+{"id": "clamp", "reference": "def clamp(x, lo, hi): ...", "tests": ["assert clamp(5, 0, 10) == 5"]}
+```
+
+Optional fields: `entry` (the function name, if it can't be inferred), `setup` (code run
+before the tests), `hidden` (held-out asserts you already have; they are used as extra
+answer-key inputs), `call_alias` (the name the tests call the solution by, e.g. `candidate`).
+
+```
+gameability check examples/tiny.jsonl --fixes fixes.py --json report.json
+```
+
+## What the number means, and what it doesn't
+
+- **The attacker is weak.** One small edit is the simplest possible way to be wrong; a model trained
+  against these tests can search far more. In that sense the rates are a floor.
+- **Fresh inputs can still break a problem's rules in ways the tests don't reveal.** v0.2 keeps each
+  argument inside its observed domain, but it does not model relationships BETWEEN arguments (a
+  length argument that must equal a list's length) or rules that only the docstring states. On the
+  problems EvalPlus covers, 4.8% of flagged MBPP and 15.4% of flagged HumanEval problems are still
+  unconfirmed (v0.1: 14.3% and 21.4%).
+- **It does not predict where a model will cheat.** On a 1.5B coder, false passes were no more
+  common on flagged problems than on others (8.4% vs 10.0%). It audits the tests' strength, not a
+  model's behaviour.
+- **It audits the tests, not any model.** Nothing here says a particular model cheated. It says
+  which problems would not have caught it.
+- **It runs the benchmark's code.** Reference solutions, mutants and tests are executed, each job in
+  its own throwaway directory (some benchmark tasks write files; v0.1 let them land in your working
+  folder). That is isolation from your files, not a security sandbox: only run it on benchmarks
+  you'd be willing to execute.
+
+## Reproducing the headline numbers
+
+```
+gameability check mbpp --json mbpp.json
+gameability check humaneval --json humaneval.json
+```
+
+Defaults: 24 mutants per problem, 8 fresh inputs, seed 0, contract inputs. Add `--inputs free` to
+reproduce v0.1's numbers (MBPP 18.5%, HumanEval 11.8%). The known-answer tests run with
+`python tests/test_gameability.py` and `python tests/test_contract_inputs.py`.
+
+## License
+
+MIT

gameability-0.2.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+gameability/__init__.py,sha256=rIBy1Ta8EceZbcm1O-I7qdZe1MeInSAalhlIPZ1l66c,840
+gameability/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
+gameability/_worker.py,sha256=JS1d-Ues0E8BlInXLRcgZKpIfizK_-kMazjUzwFwW3I,2666
+gameability/cli.py,sha256=LqHBdalFFVOeYmD4zTGh12cMAIu-sthosoK_MH3WKJs,5787
+gameability/engine.py,sha256=qYxEriBrUZBnX7Rxsbjj-kNUbQspVjUWVow6qufbVI4,12665
+gameability/harden.py,sha256=5pjtmXZIUfw2ilCNB2s2D-C6LtuIBt7Mh2My2K4Bsoc,8773
+gameability/loaders.py,sha256=LmauU6pYBEN82QkcEywyk_JQ8gB6M0Oqx-xBbDwomHs,12540
+gameability/mutate.py,sha256=nU7prNSuVW_3DOL_WURQElNzwdEQbOTall_-Lv-7jDM,5842
+gameability/oracle.py,sha256=UEgBS-jP0Uf4bjGpqaSzDH2qtRgjA6zrDC7RMX5jltQ,9965
+gameability-0.2.0.dist-info/licenses/LICENSE,sha256=JSp4X4Yy5hzOBeIJRmEHwSi6QEJm17VINS1vdQRvq_s,1078
+gameability-0.2.0.dist-info/METADATA,sha256=MKWcFo7EShcMOPzB_TJqrzfTRWPoGAeRIKStpP0nFZU,8692
+gameability-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+gameability-0.2.0.dist-info/entry_points.txt,sha256=6TNGGnuquMJ-OBgCkbaLcQnP5K5MrcLh52X4rmS4Smc,53
+gameability-0.2.0.dist-info/top_level.txt,sha256=L7ZC-VXXQ-SeaSz6gYLDHN9ZnudBeRRMs_6H1U0T3PU,12
+gameability-0.2.0.dist-info/RECORD,,

gameability-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

gameability-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gameability = gameability.cli:main

gameability-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Empyrean Intelligence
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gameability-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+gameability