agent-skill-eval 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_skill_eval-0.2.1.dist-info/METADATA +335 -0
- agent_skill_eval-0.2.1.dist-info/RECORD +15 -0
- agent_skill_eval-0.2.1.dist-info/WHEEL +4 -0
- agent_skill_eval-0.2.1.dist-info/entry_points.txt +2 -0
- agent_skill_eval-0.2.1.dist-info/licenses/LICENSE +21 -0
- skill_eval/__init__.py +8 -0
- skill_eval/__main__.py +3 -0
- skill_eval/cli.py +844 -0
- skill_eval/git_state.py +227 -0
- skill_eval/graders/__init__.py +810 -0
- skill_eval/harnesses/__init__.py +337 -0
- skill_eval/models.py +146 -0
- skill_eval/runner.py +424 -0
- skill_eval/skills.py +88 -0
- skill_eval/workspace.py +101 -0
skill_eval/git_state.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import subprocess
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
from urllib.parse import urlparse
|
|
8
|
+
|
|
9
|
+
from skill_eval.models import GitStateSnapshot
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _run(args: list[str], cwd: Path) -> str:
|
|
13
|
+
result = subprocess.run(args, cwd=cwd, capture_output=True, text=True, check=False)
|
|
14
|
+
return result.stdout or ""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def github_repo_slug(source_repo: Optional[str]) -> Optional[str]:
|
|
18
|
+
"""Return ``owner/repo`` for GitHub clone URLs or existing slugs."""
|
|
19
|
+
if not source_repo:
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
repo = source_repo.strip().rstrip("/")
|
|
23
|
+
if not repo:
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
if "://" not in repo and ":" in repo:
|
|
27
|
+
host_part, path = repo.split(":", 1)
|
|
28
|
+
if host_part.split("@")[-1].lower() == "github.com":
|
|
29
|
+
return _slug_from_repo_path(path)
|
|
30
|
+
|
|
31
|
+
parsed = urlparse(repo)
|
|
32
|
+
if parsed.netloc:
|
|
33
|
+
if (parsed.hostname or "").lower() != "github.com":
|
|
34
|
+
return None
|
|
35
|
+
return _slug_from_repo_path(parsed.path)
|
|
36
|
+
|
|
37
|
+
if repo.lower().startswith("github.com/"):
|
|
38
|
+
return _slug_from_repo_path(repo[len("github.com/") :])
|
|
39
|
+
|
|
40
|
+
return _slug_from_repo_path(repo)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _slug_from_repo_path(path: str) -> Optional[str]:
|
|
44
|
+
clean = path.split("?", 1)[0].split("#", 1)[0].strip("/").removesuffix(".git")
|
|
45
|
+
parts = [part for part in clean.split("/") if part]
|
|
46
|
+
if len(parts) != 2:
|
|
47
|
+
return None
|
|
48
|
+
return f"{parts[0]}/{parts[1]}"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def capture_git_state(workspace: Path, source_repo: Optional[str] = None) -> GitStateSnapshot:
|
|
52
|
+
"""Snapshot the git and PR state of a workspace.
|
|
53
|
+
|
|
54
|
+
Captures branches, current HEAD, commit count, recent commits, full SHAs
|
|
55
|
+
for branch heads, remote branch heads, and (if a source repo is
|
|
56
|
+
configured) the list of PRs currently open on the remote.
|
|
57
|
+
"""
|
|
58
|
+
raw_branches = _run(["git", "branch", "-a"], workspace).strip().split("\n")
|
|
59
|
+
local_branches: set[str] = set()
|
|
60
|
+
remote_branches: set[str] = set()
|
|
61
|
+
for line in raw_branches:
|
|
62
|
+
b = line.strip().lstrip("* ")
|
|
63
|
+
if not b or "HEAD" in b:
|
|
64
|
+
continue
|
|
65
|
+
if b.startswith("remotes/"):
|
|
66
|
+
stripped = b[len("remotes/") :]
|
|
67
|
+
if "/" in stripped:
|
|
68
|
+
remote_branches.add(stripped.split("/", 1)[1])
|
|
69
|
+
else:
|
|
70
|
+
local_branches.add(b)
|
|
71
|
+
|
|
72
|
+
current = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"], workspace).strip()
|
|
73
|
+
head = _run(["git", "rev-parse", "HEAD"], workspace).strip()
|
|
74
|
+
count_raw = _run(["git", "rev-list", "--all", "--count"], workspace).strip()
|
|
75
|
+
|
|
76
|
+
sha_log = _run(["git", "log", "--all", "--pretty=format:%H"], workspace).strip()
|
|
77
|
+
commit_shas = [line for line in sha_log.split("\n") if line.strip()]
|
|
78
|
+
|
|
79
|
+
short_log = _run(["git", "log", "--all", "--pretty=format:%h %s"], workspace).strip()
|
|
80
|
+
commits = [line for line in short_log.split("\n") if line.strip()]
|
|
81
|
+
|
|
82
|
+
branch_heads = dict(
|
|
83
|
+
_parse_for_each_ref(
|
|
84
|
+
_run(
|
|
85
|
+
["git", "for-each-ref", "refs/heads", "--format=%(refname:short) %(objectname)"],
|
|
86
|
+
workspace,
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
remote_for_each = _run(
|
|
92
|
+
["git", "for-each-ref", "refs/remotes", "--format=%(refname:short) %(objectname)"],
|
|
93
|
+
workspace,
|
|
94
|
+
)
|
|
95
|
+
remote_branch_heads: dict[str, str] = {}
|
|
96
|
+
for refname, sha in _parse_for_each_ref(remote_for_each):
|
|
97
|
+
if "/" in refname:
|
|
98
|
+
short = refname.split("/", 1)[1]
|
|
99
|
+
if short != "HEAD":
|
|
100
|
+
remote_branch_heads[short] = sha
|
|
101
|
+
|
|
102
|
+
remotes_raw = _run(["git", "remote", "-v"], workspace).strip()
|
|
103
|
+
remote_names = sorted({line.split()[0] for line in remotes_raw.split("\n") if line.strip()})
|
|
104
|
+
|
|
105
|
+
open_prs: list[dict] = []
|
|
106
|
+
if source_repo:
|
|
107
|
+
slug = github_repo_slug(source_repo)
|
|
108
|
+
if slug:
|
|
109
|
+
prs_raw = _run(
|
|
110
|
+
[
|
|
111
|
+
"gh",
|
|
112
|
+
"pr",
|
|
113
|
+
"list",
|
|
114
|
+
"--repo",
|
|
115
|
+
slug,
|
|
116
|
+
"--state",
|
|
117
|
+
"all",
|
|
118
|
+
"--json",
|
|
119
|
+
"number,headRefName,baseRefName,url,state",
|
|
120
|
+
],
|
|
121
|
+
Path.cwd(),
|
|
122
|
+
)
|
|
123
|
+
if prs_raw.strip():
|
|
124
|
+
try:
|
|
125
|
+
open_prs = json.loads(prs_raw)
|
|
126
|
+
except json.JSONDecodeError:
|
|
127
|
+
open_prs = []
|
|
128
|
+
|
|
129
|
+
return GitStateSnapshot(
|
|
130
|
+
local_branches=sorted(local_branches),
|
|
131
|
+
remote_branches=sorted(remote_branches),
|
|
132
|
+
current_branch=current,
|
|
133
|
+
head_sha=head,
|
|
134
|
+
commit_count=int(count_raw) if count_raw.isdigit() else len(commits),
|
|
135
|
+
commits=commits[:20],
|
|
136
|
+
remote_names=remote_names,
|
|
137
|
+
open_prs=open_prs,
|
|
138
|
+
branch_heads=branch_heads,
|
|
139
|
+
remote_branch_heads=remote_branch_heads,
|
|
140
|
+
commit_shas=commit_shas[:50],
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _parse_for_each_ref(raw: str) -> list[tuple[str, str]]:
|
|
145
|
+
out: list[tuple[str, str]] = []
|
|
146
|
+
for line in raw.split("\n"):
|
|
147
|
+
line = line.strip()
|
|
148
|
+
if not line:
|
|
149
|
+
continue
|
|
150
|
+
parts = line.rsplit(" ", 1)
|
|
151
|
+
if len(parts) == 2:
|
|
152
|
+
out.append((parts[0], parts[1]))
|
|
153
|
+
return out
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def state_diff(pre: GitStateSnapshot, post: GitStateSnapshot) -> dict:
|
|
157
|
+
"""Compute the delta between two state snapshots.
|
|
158
|
+
|
|
159
|
+
Returns a dict of the form::
|
|
160
|
+
|
|
161
|
+
{
|
|
162
|
+
"new_branches": [...],
|
|
163
|
+
"new_remote_branches": [...],
|
|
164
|
+
"current_branch": str,
|
|
165
|
+
"current_branch_changed": bool,
|
|
166
|
+
"head_advanced": bool,
|
|
167
|
+
"new_commits": [...],
|
|
168
|
+
"new_commit_shas": [...],
|
|
169
|
+
"new_open_prs": [...],
|
|
170
|
+
"eval_branch": str | None,
|
|
171
|
+
"eval_branch_pushed": bool,
|
|
172
|
+
"eval_branch_pushed_matches_head": bool,
|
|
173
|
+
}
|
|
174
|
+
"""
|
|
175
|
+
pre_local = set(pre.local_branches)
|
|
176
|
+
pre_remote = set(pre.remote_branches)
|
|
177
|
+
pre_pr_numbers = {p.get("number") for p in pre.open_prs}
|
|
178
|
+
|
|
179
|
+
new_branches = sorted(set(post.local_branches) - pre_local)
|
|
180
|
+
new_remote_branches = sorted(set(post.remote_branches) - pre_remote)
|
|
181
|
+
|
|
182
|
+
pre_head = pre.head_sha
|
|
183
|
+
post_head = post.head_sha
|
|
184
|
+
head_advanced = bool(pre_head) and bool(post_head) and pre_head != post_head
|
|
185
|
+
|
|
186
|
+
pre_sha_set = set(pre.commit_shas)
|
|
187
|
+
new_commit_shas = [s for s in post.commit_shas if s and s not in pre_sha_set]
|
|
188
|
+
|
|
189
|
+
new_commits = [c for c in post.commits if c not in pre.commits]
|
|
190
|
+
|
|
191
|
+
new_open_prs = [p for p in post.open_prs if p.get("number") not in pre_pr_numbers]
|
|
192
|
+
|
|
193
|
+
eval_branch: str | None = None
|
|
194
|
+
if new_branches:
|
|
195
|
+
if post.current_branch and post.current_branch in new_branches:
|
|
196
|
+
eval_branch = post.current_branch
|
|
197
|
+
else:
|
|
198
|
+
eval_branch = new_branches[0]
|
|
199
|
+
elif post.current_branch != pre.current_branch and post.current_branch:
|
|
200
|
+
pre_local_set = set(pre.local_branches)
|
|
201
|
+
pre_remote_set = set(pre.remote_branches)
|
|
202
|
+
if post.current_branch in pre_local_set or post.current_branch in pre_remote_set:
|
|
203
|
+
eval_branch = None
|
|
204
|
+
else:
|
|
205
|
+
eval_branch = post.current_branch
|
|
206
|
+
|
|
207
|
+
eval_branch_pushed = False
|
|
208
|
+
eval_branch_pushed_matches_head = False
|
|
209
|
+
if eval_branch and eval_branch in post.remote_branch_heads:
|
|
210
|
+
eval_branch_pushed = True
|
|
211
|
+
pushed_sha = post.remote_branch_heads.get(eval_branch)
|
|
212
|
+
if pushed_sha == post_head:
|
|
213
|
+
eval_branch_pushed_matches_head = True
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
"new_branches": new_branches,
|
|
217
|
+
"new_remote_branches": new_remote_branches,
|
|
218
|
+
"current_branch": post.current_branch,
|
|
219
|
+
"current_branch_changed": pre.current_branch != post.current_branch,
|
|
220
|
+
"head_advanced": head_advanced,
|
|
221
|
+
"new_commits": new_commits,
|
|
222
|
+
"new_commit_shas": new_commit_shas,
|
|
223
|
+
"new_open_prs": new_open_prs,
|
|
224
|
+
"eval_branch": eval_branch,
|
|
225
|
+
"eval_branch_pushed": eval_branch_pushed,
|
|
226
|
+
"eval_branch_pushed_matches_head": eval_branch_pushed_matches_head,
|
|
227
|
+
}
|