whichagent 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
whichagent/git.py ADDED
@@ -0,0 +1,137 @@
1
+ """git: history, leak-free base clones, workspaces, diffs."""
2
+ import os
3
+ import re
4
+ import shutil
5
+ import stat
6
+ import subprocess
7
+ import sys
8
+ import tempfile
9
+ import threading
10
+ from pathlib import Path
11
+
12
+ # Untracked files an agent leaves by running tools: not part of its change. Tracked files are unaffected.
13
+ EXCLUDE = """.aider*
14
+ __pycache__/
15
+ *.py[cod]
16
+ .pytest_cache/
17
+ .mypy_cache/
18
+ .ruff_cache/
19
+ .coverage
20
+ node_modules/
21
+ """
22
+ _refs = threading.Lock()
23
+
24
+
25
+ def git(cwd, *args):
26
+ r = subprocess.run(["git", "-c", "core.quotepath=off", *args], cwd=cwd, capture_output=True,
27
+ text=True, encoding="utf-8", errors="replace")
28
+ if r.returncode:
29
+ raise RuntimeError(f"git {' '.join(args[:3])}: {r.stderr.strip()[-500:]}")
30
+ return r.stdout
31
+
32
+
33
+ def show(repo, sha, path):
34
+ """File content at a commit, or None if it doesn't exist there."""
35
+ r = subprocess.run(["git", "show", f"{sha}:{path}"], cwd=repo, capture_output=True)
36
+ return r.stdout if r.returncode == 0 else None
37
+
38
+
39
+ def branch(repo):
40
+ return git(repo, "branch", "--show-current").strip() or "main"
41
+
42
+
43
+ def files(text):
44
+ """{path: [status, added, deleted]} from --raw --numstat output."""
45
+ out = {}
46
+ for line in text.splitlines():
47
+ if line.startswith(":"):
48
+ meta, path = line.split("\t", 1)
49
+ out.setdefault(path, ["M", 0, 0])[0] = meta.split()[-1][0]
50
+ elif m := re.fullmatch(r"(\d+|-)\t(\d+|-)\t(.+)", line):
51
+ f = out.setdefault(m[3], ["M", 0, 0])
52
+ f[1], f[2] = (int(x) if x != "-" else 0 for x in m.group(1, 2))
53
+ return out
54
+
55
+
56
+ def commits(repo, since):
57
+ """Non-merge commits on the current branch, newest first."""
58
+ out = git(repo, "log", "HEAD", "--no-merges", "--no-renames", f"--since={since}",
59
+ "--format=%x1e%H%x1f%P%x1f%cs%x1f%B%x1f", "--raw", "--numstat")
60
+ for chunk in out.split("\x1e")[1:]:
61
+ sha, parents, date, msg, rest = chunk.split("\x1f")
62
+ yield {"sha": sha, "parents": parents.split(), "date": date, "msg": msg.strip(), "files": files(rest)}
63
+
64
+
65
+ def base_clone(repo, sha, dest, depth):
66
+ """Bare shallow clone that ends at sha: the fix and every later commit never enter it."""
67
+ tmp = f"whichagent/tmp-{sha[:12]}"
68
+ with _refs: # ref updates in your repo, one at a time
69
+ if dest.exists():
70
+ return dest
71
+ dest.parent.mkdir(parents=True, exist_ok=True)
72
+ git(repo, "branch", "-f", tmp, sha)
73
+ try:
74
+ git(repo, "clone", "-q", "--bare", "--single-branch", "--no-tags", f"--depth={depth}",
75
+ "--branch", tmp, Path(repo).as_uri(), str(dest))
76
+ finally:
77
+ git(repo, "branch", "-D", tmp)
78
+ return dest
79
+
80
+
81
+ def workspace(bare, branch_name, name):
82
+ """Fresh clone of a base in the system temp dir, on your default branch, with no remote."""
83
+ ws = Path(tempfile.mkdtemp()) / name
84
+ git(ws.parent, "clone", "-q", str(bare), str(ws))
85
+ if git(ws, "branch", "--show-current").strip() != branch_name:
86
+ git(ws, "branch", "-m", branch_name)
87
+ git(ws, "remote", "remove", "origin")
88
+ rmtree(ws / ".git" / "logs") # the reflog names the base clone's path
89
+ (ws / ".git" / "info").mkdir(exist_ok=True)
90
+ (ws / ".git" / "info" / "exclude").write_text(EXCLUDE)
91
+ return ws
92
+
93
+
94
+ def exclude_untracked(ws):
95
+ """Keep what setup created (venvs, build output) out of the agent's diff."""
96
+ new = git(ws, "ls-files", "--others", "--exclude-standard", "--directory").splitlines()
97
+ with open(ws / ".git" / "info" / "exclude", "a", encoding="utf-8") as f:
98
+ f.writelines(f"/{p}\n" for p in new)
99
+
100
+
101
+ def snapshot(ws):
102
+ """Tree id of the workspace as the agent gets it, left unstaged."""
103
+ git(ws, "add", "-A")
104
+ tree = git(ws, "write-tree").strip()
105
+ git(ws, "reset", "-q")
106
+ return tree
107
+
108
+
109
+ def changes(ws, tree):
110
+ """Stage everything; {path: [status, added, deleted]} against tree."""
111
+ git(ws, "add", "-A")
112
+ return files(git(ws, "diff", "--cached", "--no-renames", "--raw", "--numstat", tree))
113
+
114
+
115
+ def diff(ws, tree):
116
+ return git(ws, "diff", "--cached", "--no-renames", "--no-color", "--no-ext-diff", tree)
117
+
118
+
119
+ def put(repo, sha, paths, ws):
120
+ """Make paths in ws match commit sha of repo, deleting the ones absent there."""
121
+ for p in paths:
122
+ data, f = show(repo, sha, p), Path(ws, p)
123
+ if data is None:
124
+ f.unlink(missing_ok=True)
125
+ else:
126
+ f.parent.mkdir(parents=True, exist_ok=True)
127
+ f.write_bytes(data)
128
+
129
+
130
+ def rmtree(path):
131
+ def retry(func, p, _):
132
+ os.chmod(p, stat.S_IWRITE) # git marks objects read-only
133
+ func(p)
134
+ try:
135
+ shutil.rmtree(path, **({"onexc": retry} if sys.version_info >= (3, 12) else {"onerror": retry}))
136
+ except OSError:
137
+ pass # ponytail: a file still held open (Windows) leaves the dir to the OS temp cleanup
whichagent/grade.py ADDED
@@ -0,0 +1,85 @@
1
+ """Grade a finished run: capture the diff, measure it, reset the tests, run the real ones."""
2
+ import os
3
+ import re
4
+ import shlex
5
+
6
+ from . import config as c, git as g, testresults
7
+
8
+ RUNNABLE = re.compile(
9
+ r"(^|/)(test_[^/]*\.py|[^/]*_test\.py" # pytest, unittest
10
+ r"|[^/]*\.(test|spec)\.[cm]?[jt]sx?|__tests__/[^/]*\.[cm]?[jt]sx?" # vitest, jest, node --test
11
+ r"|[^/]*_test\.go|tests/[^/]*\.rs" # go test, cargo integration tests
12
+ r"|[^/]*Tests?\.(java|kt)|Test[^/]*\.(java|kt))$") # JUnit via maven or gradle
13
+
14
+
15
+ def runnable(paths):
16
+ """Test files a runner can take as arguments: not fixtures, conftest or helpers."""
17
+ return [p for p in paths if RUNNABLE.search(p)]
18
+
19
+
20
+ def _quote(s):
21
+ return shlex.quote(s) if os.name != "nt" else f'"{s}"' if " " in s else s
22
+
23
+
24
+ def command(template, files, report=""):
25
+ """template with {files}, {dirs} (./dir per file, for go test), {names} (file stems, for cargo --test and
26
+ JUnit classes) and {report} filled in. {names:,} joins with "," instead of " "."""
27
+ lists = {"files": files,
28
+ "dirs": list(dict.fromkeys("./" + p.rsplit("/", 1)[0] if "/" in p else "." for p in files)),
29
+ "names": list(dict.fromkeys(p.rsplit("/", 1)[-1].split(".")[0] for p in files))}
30
+ cmd = c.PLACEHOLDER.sub(lambda m: (" " if m[2] is None else m[2]).join(map(_quote, lists[m[1]])), template)
31
+ return cmd.replace("{report}", _quote(report))
32
+
33
+
34
+ def test(cfg, box, ws, files, log, suite=False):
35
+ """Run the task's test files, or the whole suite. Returns (exit code or None, seconds, {test: passed} or None)."""
36
+ rp = cfg["repo"]
37
+ template = rp["suite"] or c.PLACEHOLDER.sub("", rp["test"]) if suite else rp["test"]
38
+ report = ws.parent / f"{log.stem}.report" # outside the workspace, inside what docker mode mounts
39
+ fixed = ws / rp["report"] if rp["report"] else None
40
+ for stale in (report, fixed):
41
+ if stale and stale.is_dir():
42
+ g.rmtree(stale)
43
+ elif stale:
44
+ stale.unlink(missing_ok=True)
45
+ rc, secs = box.shell(command(template, files, box.path(report)), c.secs(rp["test_timeout"]), log)
46
+ found = report if report.exists() else fixed
47
+ return rc, secs, testresults.parse(found, log.read_text("utf-8", "replace"), box.roots)
48
+
49
+
50
+ def grade(cfg, task, ws, box, start, run_dir):
51
+ """Measure the agent's changes against the start tree, then test with the real tests in place."""
52
+ root = cfg["root"]
53
+ changed = g.changes(ws, start)
54
+ (run_dir / "agent.diff").write_text(g.diff(ws, start), "utf-8")
55
+ tests = {p: f for p, f in changed.items() if c.is_test(cfg, p)}
56
+ r = {"files_changed": len(changed),
57
+ "extra_files": sum(p not in tests and p not in task["source_files"] for p in changed),
58
+ "wrote_tests": any(f[1] > 0 and not c.match(p, c.TEST_CONFIG) for p, f in tests.items()),
59
+ # removing lines from an existing test, or touching test config, counts; adding tests doesn't
60
+ "tampered": any(f[0] != "A" and f[2] > 0 or f[0] == "D" or c.match(p, c.TEST_CONFIG)
61
+ for p, f in tests.items()),
62
+ "flagged": [p for p in changed if c.match(p, c.FLAGGED)]}
63
+ g.put(root, task["base"], list(tests), ws)
64
+ g.put(root, task["fix"], task["test_files"], ws)
65
+ rc, _, results = test(cfg, box, ws, runnable(task["test_files"]), run_dir / "tests.log")
66
+ f2p, p2p = task.get("fail_to_pass"), task.get("pass_to_pass") or []
67
+ if f2p and results is not None: # per test, as SWE-bench does: the fixed tests pass, the others still do
68
+ r["f2p"] = f"{sum(results.get(t) is True for t in f2p)}/{len(f2p)}"
69
+ r["p2p"] = f"{sum(results.get(t) is True for t in p2p)}/{len(p2p)}"
70
+ r["tests_pass"] = all(results.get(t) is True for t in f2p + p2p)
71
+ else:
72
+ r["tests_pass"] = rc == 0
73
+ r["regression"] = None # checked only when the task's tests pass
74
+ failing = task.get("suite_failing")
75
+ if r["tests_pass"] and cfg["run"]["regression"] and (failing is not None or task["suite_green"]):
76
+ rc, _, results = test(cfg, box, ws, [], run_dir / "suite.log", suite=True)
77
+ if failing is not None and results is not None: # anything that passed at the fix and fails now
78
+ broke = {t for t, ok in results.items() if not ok and t not in failing}
79
+ if broke: # a flaky test shouldn't fail the agent: it counts only if it fails twice in a row
80
+ again = test(cfg, box, ws, [], run_dir / "suite-again.log", suite=True)[2] or {}
81
+ broke = {t for t in broke if again.get(t) is not True}
82
+ r["regression"], r["broke"] = bool(broke), sorted(broke)[:20]
83
+ elif task["suite_green"]:
84
+ r["regression"] = rc != 0
85
+ return r
whichagent/mine.py ADDED
@@ -0,0 +1,159 @@
1
+ """Find fix commits that came with tests; keep the ones whose tests fail before the fix and pass after."""
2
+ import concurrent.futures as cf
3
+ import itertools
4
+ import json
5
+ import re
6
+ import shutil
7
+ import subprocess
8
+
9
+ from . import config as c, git as g, grade, sandbox as sb, store, workspace
10
+
11
+ LOAD_ERROR = re.compile(r"ImportError|ModuleNotFoundError|errors? during collection|Cannot find module"
12
+ r"|does not provide an export named|undefined: |cannot find (?:symbol|value|function)")
13
+ FIXES = re.compile(r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+#(\d+)", re.I)
14
+
15
+
16
+ def candidates(cfg, gh):
17
+ """Cheap filter over history: small source change plus runnable tests, no dependency changes."""
18
+ r, m, out = cfg["repo"], cfg["mine"], []
19
+ for cm in g.commits(cfg["root"], m["since"]):
20
+ fs = cm["files"]
21
+ tests = {p for p in fs if c.is_test(cfg, p)}
22
+ srcs = sorted(p for p in fs if p not in tests)
23
+ lines = sum(fs[p][1] + fs[p][2] for p in srcs)
24
+ alive = sorted(p for p in tests if fs[p][0] != "D")
25
+ if (len(cm["parents"]) == 1 and srcs and grade.runnable(alive)
26
+ and len(srcs) <= m["max_files"] and lines <= m["max_lines"]
27
+ and not any(c.match(p, r["skip_if_changed"]) for p in fs)
28
+ and (gh or len(cm["msg"].split()) >= m["min_statement_words"])):
29
+ out.append({**cm, "tests": alive, "srcs": srcs, "lines": lines})
30
+ if len(out) == m["max_candidates"]:
31
+ break
32
+ return out
33
+
34
+
35
+ def gh_api(root, path):
36
+ try:
37
+ r = subprocess.run(["gh", "api", path], cwd=root, capture_output=True, text=True,
38
+ encoding="utf-8", errors="replace", timeout=60)
39
+ except (OSError, subprocess.TimeoutExpired):
40
+ return None
41
+ return json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else None
42
+
43
+
44
+ def statement(root, sha, msg, min_words, gh):
45
+ """(text, source): the linked issue, else the PR, else a long enough commit message."""
46
+ if gh:
47
+ prs = [p for p in gh_api(root, f"repos/{{owner}}/{{repo}}/commits/{sha}/pulls") or []
48
+ if not p["title"].startswith("Merge ")] # a merge-forward PR says nothing about the fix
49
+ for n in FIXES.findall(" ".join([msg] + [p.get("body") or "" for p in prs])):
50
+ issue = gh_api(root, f"repos/{{owner}}/{{repo}}/issues/{n}")
51
+ if issue and "pull_request" not in issue:
52
+ return f"{issue['title']}\n\n{issue.get('body') or ''}".strip(), "issue"
53
+ if prs:
54
+ return f"{prs[0]['title']}\n\n{prs[0].get('body') or ''}".strip(), "pr"
55
+ if len(msg.split()) >= min_words:
56
+ return msg, "commit"
57
+ return None, None
58
+
59
+
60
+ def _outcome(rc0, before, rc1, after):
61
+ """(verdict, why): per test when both runs have results, else by exit code."""
62
+ if before is not None and after is not None:
63
+ f2p = sorted(t for t, ok in after.items() if ok and not before.get(t))
64
+ p2p = sorted(t for t, ok in after.items() if ok and before.get(t))
65
+ if not f2p:
66
+ return None, "tests fail after the fix" if not any(after.values()) else "tests pass before the fix"
67
+ return (f2p, p2p), None
68
+ if rc0 == 0:
69
+ return None, "tests pass before the fix"
70
+ if rc1 != 0:
71
+ return None, "tests fail after the fix"
72
+ return (None, None), None
73
+
74
+
75
+ def validate(cfg, cm, gh):
76
+ """A task dict, or {"id", "reason", "log"} when the commit doesn't make a valid task."""
77
+ root, m = cfg["root"], cfg["mine"]
78
+ tid, fix, base = cm["sha"][:7], cm["sha"], cm["parents"][0]
79
+ text, source = statement(root, fix, cm["msg"], m["min_statement_words"], gh)
80
+ if not text:
81
+ return {"id": tid, "reason": "no statement"}
82
+ try:
83
+ ws, box = workspace.create(cfg, tid, base)
84
+ except RuntimeError as e:
85
+ return {"id": tid, "reason": f"git: {e}"}
86
+ log = ws.parent / "tests.log"
87
+
88
+ def reject(reason, path=log):
89
+ tail = path.read_text("utf-8", "replace")[-2000:] if path.exists() else ""
90
+ return {"id": tid, "reason": reason, "log": tail}
91
+
92
+ try:
93
+ if not workspace.prepare(cfg, ws, box, ws.parent):
94
+ return reject("setup failed", ws.parent / ("setup.log" if (ws.parent / "setup.log").exists()
95
+ else "install.log"))
96
+ g.put(root, fix, cm["tests"], ws)
97
+ files, mode, secs, seen = grade.runnable(cm["tests"]), "hidden", 0.0, None
98
+ for i in range(m["verify_runs"]):
99
+ g.put(root, base, cm["srcs"], ws)
100
+ rc0, _, before = grade.test(cfg, box, ws, files, log)
101
+ if i == 0 and LOAD_ERROR.search(log.read_text("utf-8", "replace")):
102
+ mode = "visible" # the tests can't even load: the fix adds names they use
103
+ g.put(root, fix, cm["srcs"], ws)
104
+ rc1, secs, after = grade.test(cfg, box, ws, files, log)
105
+ verdict, why = _outcome(rc0, before, rc1, after)
106
+ if why or (i and verdict != seen):
107
+ return reject("flaky" if i else why)
108
+ seen = verdict
109
+ rc, _, suite = grade.test(cfg, box, ws, [], log, suite=True)
110
+ finally:
111
+ workspace.remove(ws, box)
112
+ task = {"id": tid, "base": base, "fix": fix, "date": cm["date"], "statement": text, "statement_source": source,
113
+ "source_files": cm["srcs"], "test_files": cm["tests"], "fix_lines": cm["lines"], "mode": mode,
114
+ "suite_green": rc == 0, "test_secs": round(secs, 1)}
115
+ if seen[0] is not None: # per-test grading: what must flip, what must keep passing, what already failed
116
+ task.update(fail_to_pass=seen[0], pass_to_pass=seen[1],
117
+ suite_failing=None if suite is None else sorted(t for t, ok in suite.items() if not ok))
118
+ return task
119
+
120
+
121
+ def mine(cfg, log=print):
122
+ """Validate candidates in parallel until max_tasks; resumable through tasks.jsonl and rejected.jsonl."""
123
+ sb.STOP.clear()
124
+ root, m, d = cfg["root"], cfg["mine"], store.state(cfg["root"])
125
+ tasks = store.read(d / "tasks.jsonl")
126
+ seen = {t["id"] for t in tasks + store.read(d / "rejected.jsonl")}
127
+ gh = shutil.which("gh") is not None and gh_api(root, "repos/{owner}/{repo}") is not None
128
+ found = candidates(cfg, gh)
129
+ todo = iter([cm for cm in found if cm["sha"][:7] not in seen])
130
+ need, jobs = m["max_tasks"] - len(tasks), cfg["run"]["jobs"]
131
+ log(f"{len(found)} candidate commits, {len(tasks)} tasks mined before"
132
+ + ("" if gh else " (gh unavailable: statements come from commit messages)"))
133
+ with cf.ThreadPoolExecutor(jobs) as ex:
134
+ running = {ex.submit(validate, cfg, cm, gh) for cm in itertools.islice(todo, jobs if need > 0 else 0)}
135
+ try:
136
+ while running:
137
+ done, running = cf.wait(running, return_when=cf.FIRST_COMPLETED)
138
+ for f in done:
139
+ res = f.result()
140
+ if "reason" in res or need <= 0:
141
+ g.rmtree(d / "bases" / f"{res['id']}.git")
142
+ if "reason" in res:
143
+ store.append(d / "rejected.jsonl", res)
144
+ log(f" skip {res['id']}: {res['reason']}")
145
+ elif need > 0:
146
+ store.append(d / "tasks.jsonl", res)
147
+ tasks.append(res)
148
+ need -= 1
149
+ graded = "per test" if res.get("fail_to_pass") else "by exit code"
150
+ log(f" task {res['id']} ({res['mode']} tests, graded {graded}, "
151
+ f"statement from {res['statement_source']})")
152
+ if need > 0:
153
+ running |= {ex.submit(validate, cfg, cm, gh) for cm in itertools.islice(todo, len(done))}
154
+ except BaseException:
155
+ sb.STOP.set()
156
+ sb.killall()
157
+ raise
158
+ log(f"{len(tasks)} task{'s' * (len(tasks) != 1)} in {d / 'tasks.jsonl'}")
159
+ return tasks