hpflow 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpflow/__init__.py +4 -0
- hpflow/__main__.py +7 -0
- hpflow/briefs.py +74 -0
- hpflow/cli.py +148 -0
- hpflow/commands.py +789 -0
- hpflow/common.py +166 -0
- hpflow/fixcheck.py +71 -0
- hpflow/proof/__init__.py +9 -0
- hpflow/proof/__main__.py +7 -0
- hpflow/proof/checks.py +288 -0
- hpflow/proof/cli.py +173 -0
- hpflow/proof/domain.py +87 -0
- hpflow/proof/generate.py +264 -0
- hpflow/proof/lean.py +183 -0
- hpflow/proof/link.py +167 -0
- hpflow/proof/mutate.py +185 -0
- hpflow/proof/pins.py +66 -0
- hpflow/proof/question.py +52 -0
- hpflow/proof/stubs.py +117 -0
- hpflow/runner.py +129 -0
- hpflow-0.3.0.dist-info/METADATA +393 -0
- hpflow-0.3.0.dist-info/RECORD +26 -0
- hpflow-0.3.0.dist-info/WHEEL +5 -0
- hpflow-0.3.0.dist-info/entry_points.txt +4 -0
- hpflow-0.3.0.dist-info/licenses/LICENSE +21 -0
- hpflow-0.3.0.dist-info/top_level.txt +1 -0
hpflow/commands.py
ADDED
|
@@ -0,0 +1,789 @@
|
|
|
1
|
+
"""The commands of hpflow. Each one reads and writes JSON files in the audit folder."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import ast
|
|
5
|
+
import re
|
|
6
|
+
import shlex
|
|
7
|
+
import shutil
|
|
8
|
+
import sys
|
|
9
|
+
import threading
|
|
10
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
14
|
+
|
|
15
|
+
from . import briefs
|
|
16
|
+
from .common import (COST_NOTE, DEFAULT_CONCURRENCY, MISSING, TOOL_DIR, StepError, UsageError, copy_tree, digest, git, is_test_file, model_id,
|
|
17
|
+
read_json, run_snippet, sh, write_json)
|
|
18
|
+
from .fixcheck import check as fix_check
|
|
19
|
+
from .runner import Runner, agent_graph
|
|
20
|
+
|
|
21
|
+
DEFAULT_TEST_CMD = "{python} -m pytest -q -p no:cacheprovider {tests}"
|
|
22
|
+
EXTRA_PATHS = ("tests", "docs", "README.md", "pyproject.toml")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Context:
|
|
26
|
+
"""The audit folder, the runner and the shared options of one call."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, out: Path, runner: Runner, redo: bool = False, concurrency: int = DEFAULT_CONCURRENCY):
|
|
29
|
+
self.out = out.resolve()
|
|
30
|
+
self.runner = runner
|
|
31
|
+
self.redo = redo
|
|
32
|
+
self.concurrency = max(1, concurrency)
|
|
33
|
+
self.lock = threading.Lock()
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def src(self) -> Path:
|
|
37
|
+
return self.out / "src"
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def home(self) -> Path:
|
|
41
|
+
return self.out / "home"
|
|
42
|
+
|
|
43
|
+
def audit(self) -> Dict[str, Any]:
|
|
44
|
+
a = read_json(self.out / "audit.json")
|
|
45
|
+
if not a:
|
|
46
|
+
raise UsageError(f"no audit in {self.out}: run `init` first")
|
|
47
|
+
return a
|
|
48
|
+
|
|
49
|
+
def record_path(self, rid: str) -> Path:
|
|
50
|
+
return self.out / "records" / f"{rid}.json"
|
|
51
|
+
|
|
52
|
+
def stored(self, rid: str) -> Optional[Dict[str, Any]]:
|
|
53
|
+
"""A stored worker result, or None when the run must happen (no record, an error record, or --redo)."""
|
|
54
|
+
if self.redo:
|
|
55
|
+
return None
|
|
56
|
+
r = read_json(self.record_path(rid))
|
|
57
|
+
if not r or r.get("error"):
|
|
58
|
+
return None
|
|
59
|
+
return r
|
|
60
|
+
|
|
61
|
+
def save(self, rid: str, record: Dict[str, Any]) -> None:
|
|
62
|
+
with self.lock:
|
|
63
|
+
write_json(self.record_path(rid), record)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def now() -> str:
|
|
67
|
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def pool_map(ctx: Context, fn: Callable[[Any], Any], items: List[Any]) -> List[Any]:
|
|
71
|
+
if not items:
|
|
72
|
+
return []
|
|
73
|
+
with ThreadPoolExecutor(max_workers=ctx.concurrency) as pool:
|
|
74
|
+
return list(pool.map(fn, items))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def costs(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
78
|
+
return {"cost_usd_api_equivalent": round(sum(r.get("cost_usd") or 0 for r in records), 4),
|
|
79
|
+
"seconds": round(sum(r.get("seconds") or 0 for r in records)), "cost_note": COST_NOTE}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------- init
|
|
83
|
+
|
|
84
|
+
def cmd_init(ctx: Context, repo: str, ref: str, paths: Optional[List[str]], python: Optional[str],
|
|
85
|
+
test_cmd: Optional[str]) -> Dict[str, Any]:
|
|
86
|
+
root = Path(repo).expanduser().resolve()
|
|
87
|
+
code, top = git(root, "rev-parse", "--show-toplevel")
|
|
88
|
+
if code != 0:
|
|
89
|
+
raise UsageError(f"{root} is not a git repository")
|
|
90
|
+
root = Path(top.strip())
|
|
91
|
+
code, commit = git(root, "rev-parse", "--verify", f"{ref}^{{commit}}")
|
|
92
|
+
if code != 0:
|
|
93
|
+
raise UsageError(f"the ref {ref} is not a commit of {root}")
|
|
94
|
+
commit = commit.strip()
|
|
95
|
+
|
|
96
|
+
def present(p: str) -> bool:
|
|
97
|
+
return git(root, "cat-file", "-e", f"{commit}:{p}")[0] == 0
|
|
98
|
+
|
|
99
|
+
if paths:
|
|
100
|
+
missing = [p for p in paths if not present(p.rstrip("/"))]
|
|
101
|
+
if missing:
|
|
102
|
+
raise UsageError(f"these paths are not in {commit[:12]}: {', '.join(missing)}")
|
|
103
|
+
paths = [p.rstrip("/") for p in paths]
|
|
104
|
+
else:
|
|
105
|
+
_, listing = git(root, "ls-tree", "--name-only", "-d", commit)
|
|
106
|
+
paths = [d for d in listing.split() if d not in EXTRA_PATHS and present(f"{d}/__init__.py")]
|
|
107
|
+
paths += [p for p in EXTRA_PATHS if present(p)]
|
|
108
|
+
if not paths:
|
|
109
|
+
raise UsageError("no package folder (a top-level folder with __init__.py) found: give --paths")
|
|
110
|
+
if python is None:
|
|
111
|
+
venv = root / ".venv" / "bin" / "python"
|
|
112
|
+
python = str(venv) if venv.exists() else sys.executable
|
|
113
|
+
old = read_json(ctx.out / "audit.json")
|
|
114
|
+
if old and old.get("commit") != commit and not ctx.redo:
|
|
115
|
+
raise UsageError(f"{ctx.out} holds an audit of {old.get('commit', '')[:12]}: use a new --out or --redo")
|
|
116
|
+
audit = {"repo": str(root), "ref": ref, "commit": commit, "paths": paths, "python": python,
|
|
117
|
+
"test_cmd": test_cmd or DEFAULT_TEST_CMD, "created": (old or {}).get("created") or now()}
|
|
118
|
+
snapshot = "kept"
|
|
119
|
+
if ctx.redo or not old or old.get("paths") != paths or not ctx.src.is_dir():
|
|
120
|
+
if ctx.src.exists():
|
|
121
|
+
shutil.rmtree(ctx.src)
|
|
122
|
+
ctx.src.mkdir(parents=True)
|
|
123
|
+
tarball = ctx.out / "src.tar"
|
|
124
|
+
code, detail = git(root, "archive", "-o", str(tarball), commit, *paths)
|
|
125
|
+
if code == 0:
|
|
126
|
+
code, detail = sh(["tar", "-x", "-f", str(tarball), "-C", str(ctx.src)])
|
|
127
|
+
tarball.unlink(missing_ok=True)
|
|
128
|
+
if code != 0:
|
|
129
|
+
raise StepError(f"git archive failed: {detail[-300:]}")
|
|
130
|
+
snapshot = "written"
|
|
131
|
+
write_json(ctx.out / "audit.json", audit)
|
|
132
|
+
return {"command": "init", "ok": True, "audit": audit, "snapshot": snapshot, "src": str(ctx.src)}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# ---------------------------------------------------------------- targets
|
|
136
|
+
|
|
137
|
+
def functions_of(source: str) -> List[Dict[str, Any]]:
|
|
138
|
+
"""The functions and methods of a module: qualified name, first line (decorators included), last line."""
|
|
139
|
+
try:
|
|
140
|
+
tree = ast.parse(source)
|
|
141
|
+
except SyntaxError:
|
|
142
|
+
return []
|
|
143
|
+
out: List[Dict[str, Any]] = []
|
|
144
|
+
|
|
145
|
+
def walk(body: List[ast.stmt], prefix: str) -> None:
|
|
146
|
+
for node in body:
|
|
147
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
148
|
+
start = min([node.lineno] + [d.lineno for d in node.decorator_list])
|
|
149
|
+
out.append({"function": prefix + node.name, "start": start, "end": node.end_lineno or node.lineno})
|
|
150
|
+
elif isinstance(node, ast.ClassDef):
|
|
151
|
+
walk(node.body, prefix + node.name + ".")
|
|
152
|
+
|
|
153
|
+
walk(tree.body, "")
|
|
154
|
+
return out
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def parse_diff(text: str) -> List[Dict[str, Any]]:
|
|
158
|
+
"""Files of a `git diff -U0`, each with the old and new path and the line ranges of its hunks."""
|
|
159
|
+
files: List[Dict[str, Any]] = []
|
|
160
|
+
cur: Optional[Dict[str, Any]] = None
|
|
161
|
+
hunk = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@")
|
|
162
|
+
for line in text.splitlines():
|
|
163
|
+
if line.startswith("diff --git "):
|
|
164
|
+
cur = {"old": None, "new": None, "old_ranges": [], "new_ranges": []}
|
|
165
|
+
files.append(cur)
|
|
166
|
+
elif cur is None:
|
|
167
|
+
continue
|
|
168
|
+
elif line.startswith("--- "):
|
|
169
|
+
cur["old"] = None if line[4:] == "/dev/null" else line[4:][2:]
|
|
170
|
+
elif line.startswith("+++ "):
|
|
171
|
+
cur["new"] = None if line[4:] == "/dev/null" else line[4:][2:]
|
|
172
|
+
else:
|
|
173
|
+
m = hunk.match(line)
|
|
174
|
+
if m:
|
|
175
|
+
for (s, c), key in (((m.group(1), m.group(2)), "old_ranges"), ((m.group(3), m.group(4)), "new_ranges")):
|
|
176
|
+
start, count = int(s), 1 if c is None else int(c)
|
|
177
|
+
cur[key].append((start, start + count - 1) if count else (start, start + 1))
|
|
178
|
+
return files
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def overlaps(fn: Dict[str, Any], ranges: List[Tuple[int, int]]) -> bool:
|
|
182
|
+
return any(a <= fn["end"] and fn["start"] <= b for a, b in ranges)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def top_of(file: str) -> str:
|
|
186
|
+
parts = Path(file).parts
|
|
187
|
+
return parts[0] if len(parts) > 1 else Path(file).stem
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def make_target(file: str, fn: Dict[str, Any], how: str) -> Dict[str, Any]:
|
|
191
|
+
return {"id": digest("target", file, fn["function"]), "file": file, "function": fn["function"], "top": top_of(file),
|
|
192
|
+
"lines": [fn["start"], fn["end"]], "length": fn["end"] - fn["start"] + 1, "source": how}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def in_paths(file: str, paths: List[str]) -> bool:
|
|
196
|
+
return any(file == p or file.startswith(p.rstrip("/") + "/") for p in paths)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def cmd_targets(ctx: Context, changed: Optional[str], functions: Optional[List[str]], files: Optional[List[str]],
|
|
200
|
+
max_lines: int, max_n: Optional[int]) -> Dict[str, Any]:
|
|
201
|
+
audit = ctx.audit()
|
|
202
|
+
repo = Path(audit["repo"])
|
|
203
|
+
candidates: List[Dict[str, Any]] = []
|
|
204
|
+
skipped: List[Dict[str, Any]] = []
|
|
205
|
+
if changed:
|
|
206
|
+
if ".." not in changed or "..." in changed:
|
|
207
|
+
raise UsageError("--changed takes A..B")
|
|
208
|
+
a, b = changed.split("..", 1)
|
|
209
|
+
b = b or "HEAD"
|
|
210
|
+
for ref in (a, b):
|
|
211
|
+
if git(repo, "rev-parse", "--verify", f"{ref}^{{commit}}")[0] != 0:
|
|
212
|
+
raise UsageError(f"the ref {ref} is not a commit of {repo}")
|
|
213
|
+
code, diff = git(repo, "diff", "-U0", "--no-color", "--no-ext-diff", a, b, "--", *audit["paths"])
|
|
214
|
+
if code != 0:
|
|
215
|
+
raise StepError(f"git diff failed: {diff[-300:]}")
|
|
216
|
+
for f in parse_diff(diff):
|
|
217
|
+
path = f["new"] or f["old"]
|
|
218
|
+
if not path or not path.endswith(".py") or not in_paths(path, audit["paths"]):
|
|
219
|
+
continue
|
|
220
|
+
new_fns = functions_of(git(repo, "show", f"{b}:{f['new']}")[1]) if f["new"] else []
|
|
221
|
+
old_fns = functions_of(git(repo, "show", f"{a}:{f['old']}")[1]) if f["old"] else []
|
|
222
|
+
if is_test_file(path):
|
|
223
|
+
hit = [fn for fn in new_fns if overlaps(fn, f["new_ranges"])]
|
|
224
|
+
for fn in hit:
|
|
225
|
+
skipped.append({"file": path, "function": fn["function"], "reason": "test file"})
|
|
226
|
+
if not hit:
|
|
227
|
+
skipped.append({"file": path, "function": None, "reason": "test file"})
|
|
228
|
+
continue
|
|
229
|
+
names = {fn["function"] for fn in new_fns}
|
|
230
|
+
for fn in old_fns:
|
|
231
|
+
if fn["function"] not in names and (f["new"] is None or overlaps(fn, f["old_ranges"])):
|
|
232
|
+
skipped.append({"file": path, "function": fn["function"], "reason": "removed"})
|
|
233
|
+
for fn in new_fns:
|
|
234
|
+
if overlaps(fn, f["new_ranges"]):
|
|
235
|
+
candidates.append(make_target(f["new"], fn, "changed"))
|
|
236
|
+
else:
|
|
237
|
+
wanted: List[Tuple[str, Optional[str]]] = []
|
|
238
|
+
for spec in functions or []:
|
|
239
|
+
if ":" not in spec:
|
|
240
|
+
raise UsageError(f"--function takes FILE:NAME, not {spec}")
|
|
241
|
+
file, name = spec.rsplit(":", 1)
|
|
242
|
+
wanted.append((file, name))
|
|
243
|
+
wanted += [(file, None) for file in files or []]
|
|
244
|
+
for file, name in wanted:
|
|
245
|
+
path = ctx.src / file
|
|
246
|
+
if not path.is_file():
|
|
247
|
+
raise UsageError(f"{file} is not in the snapshot {ctx.src}")
|
|
248
|
+
fns = functions_of(path.read_text(encoding="utf-8", errors="replace"))
|
|
249
|
+
if name is not None:
|
|
250
|
+
fns = [fn for fn in fns if fn["function"] == name]
|
|
251
|
+
if not fns:
|
|
252
|
+
raise UsageError(f"no function {name} in {file}")
|
|
253
|
+
for fn in fns:
|
|
254
|
+
candidates.append(make_target(file, fn, "function" if name else "file"))
|
|
255
|
+
seen, targets = set(), []
|
|
256
|
+
for t in sorted(candidates, key=lambda t: (t["file"], t["lines"][0])):
|
|
257
|
+
if t["id"] in seen:
|
|
258
|
+
continue
|
|
259
|
+
seen.add(t["id"])
|
|
260
|
+
if t["length"] > max_lines:
|
|
261
|
+
skipped.append({"file": t["file"], "function": t["function"],
|
|
262
|
+
"reason": f"too long ({t['length']} lines, the limit is {max_lines})"})
|
|
263
|
+
elif max_n is not None and len(targets) >= max_n:
|
|
264
|
+
skipped.append({"file": t["file"], "function": t["function"], "reason": f"over --max {max_n}"})
|
|
265
|
+
else:
|
|
266
|
+
targets.append(t)
|
|
267
|
+
data = {"mode": "changed" if changed else "named", "changed": changed, "max_lines": max_lines, "targets": targets,
|
|
268
|
+
"skipped": skipped}
|
|
269
|
+
write_json(ctx.out / "targets.json", data)
|
|
270
|
+
return {"command": "targets", "ok": True, "targets": [f"{t['file']}:{t['function']}" for t in targets],
|
|
271
|
+
"skipped": skipped}
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
# ---------------------------------------------------------------- review
|
|
275
|
+
|
|
276
|
+
def prepare_folder(ctx: Context, folder: Path) -> None:
|
|
277
|
+
if folder.exists():
|
|
278
|
+
shutil.rmtree(folder)
|
|
279
|
+
folder.mkdir(parents=True)
|
|
280
|
+
copy_tree(ctx.src, folder / "project")
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def cmd_review(ctx: Context, model: str, shell: bool, passes: int, repro_timeout: float) -> Dict[str, Any]:
|
|
284
|
+
audit = ctx.audit()
|
|
285
|
+
targets = (read_json(ctx.out / "targets.json") or {}).get("targets")
|
|
286
|
+
if targets is None:
|
|
287
|
+
raise UsageError("no targets.json: run `targets` first")
|
|
288
|
+
arm = "shell" if shell else "plain"
|
|
289
|
+
jobs = [(t, p) for t in targets for p in range(1, passes + 1)]
|
|
290
|
+
|
|
291
|
+
def one(job: Tuple[Dict[str, Any], int]) -> Tuple[Dict[str, Any], bool]:
|
|
292
|
+
t, p = job
|
|
293
|
+
rid = digest("review", audit["commit"], t["id"], p, model_id(model), arm)
|
|
294
|
+
r = ctx.stored(rid)
|
|
295
|
+
if r:
|
|
296
|
+
return r, False
|
|
297
|
+
folder = ctx.out / "runs" / rid
|
|
298
|
+
prepare_folder(ctx, folder)
|
|
299
|
+
task = briefs.REVIEW.format(function=t["function"], file=t["file"], top=t["top"], arm_text=briefs.REVIEW_ARM[arm])
|
|
300
|
+
graph = agent_graph(f"hpf-{rid}", task, ["findings.json", "finding_*.py"], model, shell=shell,
|
|
301
|
+
check=briefs.REVIEW_CHECK, timeout=2400)
|
|
302
|
+
r = ctx.runner.run(graph, folder)
|
|
303
|
+
r.update(kind="review", run=rid, folder=str(folder), target=t["id"], pass_n=p, model=model_id(model), arm=arm,
|
|
304
|
+
finished=now())
|
|
305
|
+
ctx.save(rid, r)
|
|
306
|
+
return r, True
|
|
307
|
+
|
|
308
|
+
results = pool_map(ctx, one, jobs)
|
|
309
|
+
records = [r for r, _ in results]
|
|
310
|
+
findings: List[Dict[str, Any]] = []
|
|
311
|
+
by_id = {t["id"]: t for t in targets}
|
|
312
|
+
|
|
313
|
+
def gate(r: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
314
|
+
folder = Path(r["folder"])
|
|
315
|
+
items = read_json(folder / "findings.json")
|
|
316
|
+
if not isinstance(items, list):
|
|
317
|
+
r["note"] = "no valid findings.json"
|
|
318
|
+
return []
|
|
319
|
+
out = []
|
|
320
|
+
for i, f in enumerate(items[:5]):
|
|
321
|
+
if not isinstance(f, dict):
|
|
322
|
+
continue
|
|
323
|
+
n = f.get("n", i + 1)
|
|
324
|
+
t = by_id[r["target"]]
|
|
325
|
+
row = {"id": f"{r['run']}-{n}", "run": r["run"], "n": n, "target": t["id"], "file": t["file"],
|
|
326
|
+
"function": t["function"], "model": r["model"], "claim": str(f.get("claim"))[:600],
|
|
327
|
+
"expected": str(f.get("expected"))[:300], "confidence": f.get("confidence"),
|
|
328
|
+
"repro": str(folder / f"finding_{n}.py")}
|
|
329
|
+
snippet = folder / f"finding_{n}.py"
|
|
330
|
+
if not snippet.exists():
|
|
331
|
+
row.update(grade=0, reason="no repro file")
|
|
332
|
+
else:
|
|
333
|
+
got = run_snippet(ctx.src, snippet, ctx.home, timeout=repro_timeout)
|
|
334
|
+
row["run_result"] = got
|
|
335
|
+
if got.get("ok"):
|
|
336
|
+
row.update(grade=1, result=got["result"])
|
|
337
|
+
elif got.get("error") == "timeout":
|
|
338
|
+
row.update(grade=0, reason=f"the repro timed out ({repro_timeout} seconds)")
|
|
339
|
+
elif got.get("error") in ("ImportError", "ModuleNotFoundError", "AttributeError", "NameError"):
|
|
340
|
+
row.update(grade=0, reason=f"the repro cannot import the function: {got.get('error')}: {got.get('message')}")
|
|
341
|
+
else:
|
|
342
|
+
row.update(grade=0, reason=f"run() raised {got.get('error')}: {got.get('message')}")
|
|
343
|
+
out.append(row)
|
|
344
|
+
return out
|
|
345
|
+
|
|
346
|
+
for rows in pool_map(ctx, gate, records):
|
|
347
|
+
findings.extend(rows)
|
|
348
|
+
write_json(ctx.out / "review.json", {"model": model_id(model), "arm": arm, "passes": passes,
|
|
349
|
+
"runs": [r["run"] for r in records], "findings": findings})
|
|
350
|
+
failed = [{"run": r["run"], "target": r["target"], "error": r.get("error") or r.get("note") or r.get("status")}
|
|
351
|
+
for r in records if r.get("error") or r.get("note")]
|
|
352
|
+
report = {"command": "review", "ok": not failed, "runs": len(records), "worker_runs": sum(1 for _, n in results if n),
|
|
353
|
+
"reused": sum(1 for _, n in results if not n), "findings": len(findings),
|
|
354
|
+
"grade_1": sum(1 for f in findings if f["grade"] == 1),
|
|
355
|
+
"grade_0": [{"id": f["id"], "reason": f["reason"]} for f in findings if f["grade"] == 0],
|
|
356
|
+
"failed_runs": failed}
|
|
357
|
+
report.update(costs(records))
|
|
358
|
+
return report
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
# ---------------------------------------------------------------- verify
|
|
362
|
+
|
|
363
|
+
def verdict_class(verdicts: Dict[str, Optional[str]], need: int) -> str:
|
|
364
|
+
vals = list(verdicts.values())
|
|
365
|
+
if len(vals) < need or any(v is None for v in vals):
|
|
366
|
+
return "mixed"
|
|
367
|
+
if all(v == "real" for v in vals):
|
|
368
|
+
return "confirmed"
|
|
369
|
+
if all(v in ("real", "real_unreachable") for v in vals):
|
|
370
|
+
return "real_unreachable"
|
|
371
|
+
if all(v == "intended" for v in vals):
|
|
372
|
+
return "by_design"
|
|
373
|
+
return "mixed"
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def cmd_verify(ctx: Context, models: List[str]) -> Dict[str, Any]:
|
|
377
|
+
audit = ctx.audit()
|
|
378
|
+
review = read_json(ctx.out / "review.json")
|
|
379
|
+
if review is None:
|
|
380
|
+
raise UsageError("no review.json: run `review` first")
|
|
381
|
+
if len(models) < 2:
|
|
382
|
+
raise UsageError("verify needs two models or more")
|
|
383
|
+
groups: Dict[str, Dict[str, Any]] = {}
|
|
384
|
+
for f in sorted((f for f in review["findings"] if f["grade"] >= 1), key=lambda f: (f["target"], f["run"], f["n"])):
|
|
385
|
+
g = groups.setdefault(f["target"], {"target": f["target"], "file": f["file"], "function": f["function"], "claims": []})
|
|
386
|
+
g["claims"].append(dict(f, k=len(g["claims"]) + 1))
|
|
387
|
+
jobs = [(g, m) for g in groups.values() for m in models]
|
|
388
|
+
|
|
389
|
+
def one(job: Tuple[Dict[str, Any], str]) -> Tuple[Dict[str, Any], bool]:
|
|
390
|
+
g, m = job
|
|
391
|
+
rid = digest("verify", audit["commit"], g["target"], model_id(m), *[c["id"] for c in g["claims"]])
|
|
392
|
+
r = ctx.stored(rid)
|
|
393
|
+
if r:
|
|
394
|
+
return r, False
|
|
395
|
+
folder = ctx.out / "runs" / rid
|
|
396
|
+
prepare_folder(ctx, folder)
|
|
397
|
+
(folder / ".gitignore").write_text(".lake/\n__pycache__/\n*.pyc\n", encoding="utf-8")
|
|
398
|
+
claims = []
|
|
399
|
+
for c in g["claims"]:
|
|
400
|
+
shutil.copy(c["repro"], folder / f"claim_{c['k']}.py")
|
|
401
|
+
claims.append({"k": c["k"], "claim": c["claim"], "expected_by_the_review": c["expected"],
|
|
402
|
+
"result_now": c["result"]})
|
|
403
|
+
write_json(folder / "claims.json", claims)
|
|
404
|
+
task = briefs.VERIFY.format(repo=Path(audit["repo"]).name, n=len(claims), function=g["function"], file=g["file"])
|
|
405
|
+
graph = agent_graph(f"hpf-{rid}", task, ["verdicts.json"], m, shell=True, check=briefs.VERIFY_CHECK, timeout=3000)
|
|
406
|
+
r = ctx.runner.run(graph, folder)
|
|
407
|
+
v = read_json(folder / "verdicts.json")
|
|
408
|
+
r.update(kind="verify", run=rid, folder=str(folder), target=g["target"], model=model_id(m),
|
|
409
|
+
verdicts=v if isinstance(v, list) else None, finished=now())
|
|
410
|
+
ctx.save(rid, r)
|
|
411
|
+
return r, True
|
|
412
|
+
|
|
413
|
+
results = pool_map(ctx, one, jobs)
|
|
414
|
+
records = [r for r, _ in results]
|
|
415
|
+
mids = [model_id(m) for m in models]
|
|
416
|
+
claims_out: List[Dict[str, Any]] = []
|
|
417
|
+
bugs: List[Dict[str, Any]] = []
|
|
418
|
+
for g in groups.values():
|
|
419
|
+
mine = {r["model"]: {v.get("k"): v for v in (r.get("verdicts") or []) if isinstance(v, dict)}
|
|
420
|
+
for r in records if r["target"] == g["target"]}
|
|
421
|
+
first = mine.get(mids[0], {})
|
|
422
|
+
root: Dict[int, int] = {}
|
|
423
|
+
for c in g["claims"]:
|
|
424
|
+
d = (first.get(c["k"]) or {}).get("duplicate_of")
|
|
425
|
+
root[c["k"]] = root.get(d, d) if isinstance(d, int) and d < c["k"] and d in root else c["k"]
|
|
426
|
+
rows = []
|
|
427
|
+
for c in g["claims"]:
|
|
428
|
+
details = {m: mine.get(m, {}).get(c["k"]) for m in mids}
|
|
429
|
+
verdicts = {m: (v or {}).get("verdict") for m, v in details.items()}
|
|
430
|
+
cls = verdict_class(verdicts, len(mids))
|
|
431
|
+
row = {"id": c["id"], "k": c["k"], "target": g["target"], "file": g["file"], "function": g["function"],
|
|
432
|
+
"claim": c["claim"], "result_now": c["result"], "repro": c["repro"], "verdicts": verdicts,
|
|
433
|
+
"class": cls, "grade": 2 if cls == "confirmed" else 1, "duplicate_of": root[c["k"]] if root[c["k"]] != c["k"] else None,
|
|
434
|
+
"details": details}
|
|
435
|
+
rows.append(row)
|
|
436
|
+
claims_out.extend(rows)
|
|
437
|
+
clusters: Dict[int, List[Dict[str, Any]]] = {}
|
|
438
|
+
for row in rows:
|
|
439
|
+
if row["grade"] == 2:
|
|
440
|
+
clusters.setdefault(root[row["k"]], []).append(row)
|
|
441
|
+
for key, members in clusters.items():
|
|
442
|
+
lead = next((x for x in rows if x["k"] == key and x["grade"] == 2), members[0])
|
|
443
|
+
info = lead["details"].get(mids[0]) or {}
|
|
444
|
+
bugs.append({"id": digest("bug", g["target"], *[x["id"] for x in members]), "target": g["target"],
|
|
445
|
+
"file": g["file"], "function": g["function"], "claim": lead["claim"],
|
|
446
|
+
"claims": [x["k"] for x in members], "result_now": lead["result_now"],
|
|
447
|
+
"what_goes_wrong": info.get("what_goes_wrong"), "caller": info.get("caller"),
|
|
448
|
+
"severity": info.get("severity"), "fix_idea": info.get("fix_idea"),
|
|
449
|
+
"repros": [{"finding": x["id"], "k": x["k"], "path": x["repro"], "result_now": x["result_now"]}
|
|
450
|
+
for x in members]})
|
|
451
|
+
write_json(ctx.out / "verify.json", {"models": mids, "runs": [r["run"] for r in records], "claims": claims_out})
|
|
452
|
+
write_json(ctx.out / "confirmed.json", bugs)
|
|
453
|
+
(ctx.out / "REPORT.md").write_text(report_md(audit, mids, claims_out, bugs), encoding="utf-8")
|
|
454
|
+
failed = [{"run": r["run"], "model": r["model"], "error": r.get("error") or "no verdicts.json"}
|
|
455
|
+
for r in records if r.get("error") or r.get("verdicts") is None]
|
|
456
|
+
by_class: Dict[str, int] = {}
|
|
457
|
+
for c in claims_out:
|
|
458
|
+
by_class[c["class"]] = by_class.get(c["class"], 0) + 1
|
|
459
|
+
report = {"command": "verify", "ok": not failed, "groups": len(groups), "claims": len(claims_out),
|
|
460
|
+
"worker_runs": sum(1 for _, n in results if n), "reused": sum(1 for _, n in results if not n),
|
|
461
|
+
"classes": by_class, "confirmed_bugs": len(bugs), "failed_runs": failed,
|
|
462
|
+
"report": str(ctx.out / "REPORT.md")}
|
|
463
|
+
report.update(costs(records))
|
|
464
|
+
return report
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def cell(x: Any) -> str:
|
|
468
|
+
return str("" if x is None else x).replace("|", "\\|").replace("\n", " ")[:300]
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def report_md(audit: Dict[str, Any], models: List[str], claims: List[Dict[str, Any]], bugs: List[Dict[str, Any]]) -> str:
|
|
472
|
+
lines = [f"# hpflow audit of {Path(audit['repo']).name} at {audit['commit'][:12]}", "",
|
|
473
|
+
f"Verifiers: {', '.join(models)}. A claim is confirmed (grade 2) when each verifier says `real`.",
|
|
474
|
+
"Grade 2 is the agreement of two agents, not a proof. The owner of the code decides what is a bug.", "",
|
|
475
|
+
f"Confirmed bugs after duplicates fold: {len(bugs)}.", ""]
|
|
476
|
+
titles = [("confirmed", "Confirmed"), ("real_unreachable", "Real but not reachable"), ("by_design", "By design"),
|
|
477
|
+
("mixed", "Mixed")]
|
|
478
|
+
for cls, title in titles:
|
|
479
|
+
rows = [c for c in claims if c["class"] == cls]
|
|
480
|
+
lines += [f"## {title} ({len(rows)})", ""]
|
|
481
|
+
if not rows:
|
|
482
|
+
lines += ["None.", ""]
|
|
483
|
+
continue
|
|
484
|
+
lines += ["| Function | Claim | Same as | Verdicts | Result now | What goes wrong | Caller |",
|
|
485
|
+
"| --- | --- | --- | --- | --- | --- | --- |"]
|
|
486
|
+
for c in rows:
|
|
487
|
+
info = c["details"].get(models[0]) or {}
|
|
488
|
+
verdicts = ", ".join(f"{m}: {v}" for m, v in c["verdicts"].items())
|
|
489
|
+
lines.append(f"| `{cell(c['file'])}:{cell(c['function'])}` k={c['k']} | {cell(c['claim'])} | "
|
|
490
|
+
f"{cell(c['duplicate_of'])} | {cell(verdicts)} | `{cell(c['result_now'])}` | "
|
|
491
|
+
f"{cell(info.get('what_goes_wrong'))} | {cell(info.get('caller'))} |")
|
|
492
|
+
lines.append("")
|
|
493
|
+
return "\n".join(lines)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
# ---------------------------------------------------------------- fix
|
|
497
|
+
|
|
498
|
+
def slug_of(file: str) -> str:
|
|
499
|
+
stem = file[:-3] if file.endswith(".py") else file
|
|
500
|
+
return re.sub(r"[^a-z0-9]+", "-", stem.lower()).strip("-")
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def bug_text(bugs: List[Dict[str, Any]]) -> str:
|
|
504
|
+
rows = []
|
|
505
|
+
for b in bugs:
|
|
506
|
+
rows.append(f"- Bug `{b['id']}` in `{b['function']}` (claims {', '.join(str(k) for k in b['claims'])}).\n"
|
|
507
|
+
f" What goes wrong: {b['what_goes_wrong']}\n The review said: {b['claim']}\n"
|
|
508
|
+
f" Result now: {b['result_now']}\n A caller that reaches it: {b['caller']}\n"
|
|
509
|
+
f" Fix idea of a reviewer (not binding): {b['fix_idea']}")
|
|
510
|
+
return "\n".join(rows)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def fix_graph(ctx: Context, audit: Dict[str, Any], g: Dict[str, Any], base: str, info: Path, model: str,
|
|
514
|
+
judge_model: str, rules: str) -> Dict[str, Any]:
|
|
515
|
+
py = audit["python"]
|
|
516
|
+
check = (f"PYTHONPATH={shlex.quote(str(TOOL_DIR))} {shlex.quote(sys.executable)} -m hpflow.fixcheck "
|
|
517
|
+
f"{shlex.quote(str(info))}")
|
|
518
|
+
test_run = audit["test_cmd"].format(python=py, tests=g["test_file"])
|
|
519
|
+
fmt = dict(repo=Path(audit["repo"]).name, base=base, n=len(g["bugs"]), file=g["file"], bugs=bug_text(g["bugs"]),
|
|
520
|
+
info=info, test_file=g["test_file"], test_run=test_run, rules=rules, check=check)
|
|
521
|
+
tools = ["Read", "Glob", "Grep", "Write", "Edit"] + [f"Bash({c}:*)" for c in (
|
|
522
|
+
"cd", "ls", "pwd", "cat", "git diff", "git status", "git log", "git show", "git grep", "git blame", "grep", "rg",
|
|
523
|
+
"wc", "head", "tail", "sed", "find", "python3", py, "sort", "uniq", "diff", "echo", "test", "mkdir", "timeout")]
|
|
524
|
+
payload = {"allowed_tools": tools, "disallowed_tools": ["WebFetch", "WebSearch", "Task", "Agent", "NotebookEdit"]}
|
|
525
|
+
files = [f"{p}/**" if (ctx.src / p).is_dir() else p for p in audit["paths"]]
|
|
526
|
+
if "tests/**" not in files:
|
|
527
|
+
files.append("tests/**")
|
|
528
|
+
return {"id": f"hpf-{digest('fixflow', g['slug'], base)}", "entry": "build",
|
|
529
|
+
"defaults": {"adapter": "claude-code", "model": model_id(model), "timeout_seconds": 5400, "finish": "criteria"},
|
|
530
|
+
"nodes": [
|
|
531
|
+
{"id": "build", "kind": "agent", "role": "code", "files": files, "task": briefs.BUILD.format(**fmt),
|
|
532
|
+
"payload": payload},
|
|
533
|
+
{"id": "check", "kind": "shell", "command": check, "timeoutMs": 2700000},
|
|
534
|
+
{"id": "review", "kind": "judge", "model": model_id(judge_model), "task": briefs.JUDGE.format(**fmt),
|
|
535
|
+
"payload": payload},
|
|
536
|
+
{"id": "done", "kind": "end", "status": "pass"},
|
|
537
|
+
{"id": "failed", "kind": "end", "status": "fail"}],
|
|
538
|
+
"edges": [{"from": "build", "to": "check", "when": "ok"},
|
|
539
|
+
{"from": "check", "to": "review", "when": "ok"},
|
|
540
|
+
{"from": "check", "to": "build", "when": "fail", "max": 2},
|
|
541
|
+
{"from": "review", "to": "done", "when": "PASS"},
|
|
542
|
+
{"from": "review", "to": "build", "when": "FAIL", "max": 2},
|
|
543
|
+
{"from": "review", "to": "build", "when": "PARTIAL", "max": 1},
|
|
544
|
+
{"from": "review", "to": "done", "when": "PARTIAL"},
|
|
545
|
+
{"from": "check", "to": "failed"}, {"from": "build", "to": "failed"},
|
|
546
|
+
{"from": "review", "to": "failed"}]}
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def cmd_fix(ctx: Context, base: Optional[str], only: Optional[List[str]], model: str, judge_model: str,
|
|
550
|
+
rules: Optional[str]) -> Dict[str, Any]:
|
|
551
|
+
audit = ctx.audit()
|
|
552
|
+
repo = Path(audit["repo"])
|
|
553
|
+
bugs = read_json(ctx.out / "confirmed.json")
|
|
554
|
+
if bugs is None:
|
|
555
|
+
raise UsageError("no confirmed.json: run `verify` first")
|
|
556
|
+
if only:
|
|
557
|
+
unknown = [b for b in only if b not in {x["id"] for x in bugs}]
|
|
558
|
+
if unknown:
|
|
559
|
+
raise UsageError(f"no confirmed bug with the id {', '.join(unknown)}")
|
|
560
|
+
bugs = [b for b in bugs if b["id"] in only]
|
|
561
|
+
base_ref = base or audit["commit"]
|
|
562
|
+
code, base_commit = git(repo, "rev-parse", "--verify", f"{base_ref}^{{commit}}")
|
|
563
|
+
if code != 0:
|
|
564
|
+
raise UsageError(f"the base {base_ref} is not a commit of {repo}")
|
|
565
|
+
base_commit = base_commit.strip()
|
|
566
|
+
groups: Dict[str, Dict[str, Any]] = {}
|
|
567
|
+
for b in bugs:
|
|
568
|
+
s = slug_of(b["file"])
|
|
569
|
+
g = groups.setdefault(s, {"slug": s, "file": b["file"],
|
|
570
|
+
"test_file": f"tests/test_hpfix_{s.replace('-', '_')}.py", "bugs": []})
|
|
571
|
+
g["bugs"].append(b)
|
|
572
|
+
|
|
573
|
+
def one(g: Dict[str, Any]) -> Tuple[Dict[str, Any], bool]:
|
|
574
|
+
rid = digest("fix", audit["commit"], base_commit, g["slug"], *sorted(b["id"] for b in g["bugs"]))
|
|
575
|
+
tree = ctx.out / "fix" / g["slug"]
|
|
576
|
+
branch = f"hpfix/{g['slug']}"
|
|
577
|
+
r = ctx.stored(rid)
|
|
578
|
+
if r and tree.exists():
|
|
579
|
+
return r, False
|
|
580
|
+
info = ctx.out / "fixinfo" / g["slug"]
|
|
581
|
+
with ctx.lock:
|
|
582
|
+
if tree.exists():
|
|
583
|
+
git(repo, "worktree", "remove", "--force", str(tree))
|
|
584
|
+
shutil.rmtree(tree, ignore_errors=True)
|
|
585
|
+
git(repo, "worktree", "prune")
|
|
586
|
+
git(repo, "branch", "-D", branch)
|
|
587
|
+
tree.parent.mkdir(parents=True, exist_ok=True)
|
|
588
|
+
code, out = git(repo, "worktree", "add", "-q", "-b", branch, str(tree), base_commit)
|
|
589
|
+
result: Dict[str, Any] = {"kind": "fix", "run": rid, "slug": g["slug"], "file": g["file"], "branch": branch,
|
|
590
|
+
"tree": str(tree), "base": base_commit, "bugs": [b["id"] for b in g["bugs"]],
|
|
591
|
+
"test_file": g["test_file"]}
|
|
592
|
+
if code != 0:
|
|
593
|
+
result.update(status="error", error="worktree", detail=out[-300:])
|
|
594
|
+
ctx.save(rid, result)
|
|
595
|
+
return result, True
|
|
596
|
+
if info.exists():
|
|
597
|
+
shutil.rmtree(info)
|
|
598
|
+
info.mkdir(parents=True)
|
|
599
|
+
spec_bugs = []
|
|
600
|
+
for b in g["bugs"]:
|
|
601
|
+
repros = []
|
|
602
|
+
for rp in b["repros"]:
|
|
603
|
+
name = f"{b['id']}_claim_{rp['k']}.py"
|
|
604
|
+
shutil.copy(rp["path"], info / name)
|
|
605
|
+
repros.append({"k": rp["k"], "file": name, "result_now": rp["result_now"]})
|
|
606
|
+
spec_bugs.append({"id": b["id"], "repros": repros})
|
|
607
|
+
write_json(info / "spec.json", {"file": g["file"], "test_file": g["test_file"], "python": audit["python"],
|
|
608
|
+
"test_cmd": audit["test_cmd"], "bugs": spec_bugs})
|
|
609
|
+
graph = fix_graph(ctx, audit, g, base_commit, info, model, judge_model, rules or briefs.DEFAULT_RULES)
|
|
610
|
+
r = ctx.runner.run(graph, tree)
|
|
611
|
+
result.update(r)
|
|
612
|
+
result["check"] = fix_check(info, tree)
|
|
613
|
+
_, stat = git(tree, "diff", "--stat")
|
|
614
|
+
_, status = git(tree, "status", "--porcelain", "--untracked-files=all")
|
|
615
|
+
result["diffstat"] = stat.strip()
|
|
616
|
+
result["new_files"] = [x[3:] for x in status.splitlines() if x.startswith("??")]
|
|
617
|
+
reviews = [s for s in r.get("steps") or [] if s.get("node") == "review"]
|
|
618
|
+
result["judge"] = reviews[-1].get("verdict") if reviews else None
|
|
619
|
+
result["finished"] = now()
|
|
620
|
+
ctx.save(rid, result)
|
|
621
|
+
return result, True
|
|
622
|
+
|
|
623
|
+
results = pool_map(ctx, one, list(groups.values()))
|
|
624
|
+
records = [r for r, _ in results]
|
|
625
|
+
write_json(ctx.out / "fix.json", {"base": base_commit, "runs": [r["run"] for r in records]})
|
|
626
|
+
rows = [{"slug": r["slug"], "worktree": r["tree"], "branch": r["branch"], "bugs": r["bugs"],
|
|
627
|
+
"flow_status": r.get("status"), "diffstat": r.get("diffstat"), "new_files": r.get("new_files"),
|
|
628
|
+
"check": (r.get("check") or {}).get("verdict"), "check_problems": (r.get("check") or {}).get("problems"),
|
|
629
|
+
"judge": r.get("judge"), "error": r.get("error")} for r in records]
|
|
630
|
+
ok = all(not r.get("error") and (r.get("check") or {}).get("verdict") == "accept" for r in records)
|
|
631
|
+
report = {"command": "fix", "ok": ok, "base": base_commit, "worktrees": rows,
|
|
632
|
+
"worker_runs": sum(1 for _, n in results if n), "reused": sum(1 for _, n in results if not n),
|
|
633
|
+
"note": "hpflow does not commit, push or merge. Review each worktree and commit it yourself."}
|
|
634
|
+
report.update(costs(records))
|
|
635
|
+
return report
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
# ---------------------------------------------------------------- status
|
|
639
|
+
|
|
640
|
+
def accepted_fixes(fix_recs: List[Dict[str, Any]]) -> set:
|
|
641
|
+
"""The bugs that have a fix with an accepted script check and a PASS judge."""
|
|
642
|
+
fixed: set = set()
|
|
643
|
+
for r in fix_recs:
|
|
644
|
+
if (r.get("check") or {}).get("verdict") == "accept" and r.get("judge") == "PASS":
|
|
645
|
+
fixed.update(r.get("bugs") or [])
|
|
646
|
+
return fixed
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def cmd_status(ctx: Context) -> Dict[str, Any]:
|
|
650
|
+
audit = ctx.audit()
|
|
651
|
+
targets = (read_json(ctx.out / "targets.json") or {}).get("targets")
|
|
652
|
+
review = read_json(ctx.out / "review.json")
|
|
653
|
+
verify = read_json(ctx.out / "verify.json")
|
|
654
|
+
confirmed = read_json(ctx.out / "confirmed.json")
|
|
655
|
+
fix = read_json(ctx.out / "fix.json")
|
|
656
|
+
|
|
657
|
+
def recs(ids: List[str]) -> List[Dict[str, Any]]:
|
|
658
|
+
return [r for r in (read_json(ctx.record_path(i)) for i in ids) if r]
|
|
659
|
+
|
|
660
|
+
steps: Dict[str, Any] = {}
|
|
661
|
+
review_recs = recs((review or {}).get("runs") or [])
|
|
662
|
+
verify_recs = recs((verify or {}).get("runs") or [])
|
|
663
|
+
fix_recs = recs((fix or {}).get("runs") or [])
|
|
664
|
+
for name, rs in (("review", review_recs), ("verify", verify_recs), ("fix", fix_recs)):
|
|
665
|
+
steps[name] = dict(runs=len(rs), **costs(rs))
|
|
666
|
+
grade: Dict[str, int] = {}
|
|
667
|
+
if review:
|
|
668
|
+
for f in review["findings"]:
|
|
669
|
+
grade[f["id"]] = f["grade"]
|
|
670
|
+
for c in (verify or {}).get("claims") or []:
|
|
671
|
+
grade[c["id"]] = max(grade.get(c["id"], 0), c["grade"])
|
|
672
|
+
fixed = accepted_fixes(fix_recs)
|
|
673
|
+
for b in confirmed or []:
|
|
674
|
+
if b["id"] in fixed:
|
|
675
|
+
for rp in b["repros"]:
|
|
676
|
+
grade[rp["finding"]] = 3
|
|
677
|
+
counts = {str(n): sum(1 for v in grade.values() if v == n) for n in range(4)}
|
|
678
|
+
open_items: List[str] = []
|
|
679
|
+
if targets is None:
|
|
680
|
+
open_items.append("targets: not run")
|
|
681
|
+
if review is None:
|
|
682
|
+
open_items.append("review: not run")
|
|
683
|
+
else:
|
|
684
|
+
done = {r["target"] for r in review_recs if not r.get("error")}
|
|
685
|
+
missing = [t for t in targets or [] if t["id"] not in done]
|
|
686
|
+
if missing:
|
|
687
|
+
open_items.append(f"review: {len(missing)} targets have no reviewer result")
|
|
688
|
+
bad = [r["run"] for r in review_recs if r.get("error")]
|
|
689
|
+
if bad:
|
|
690
|
+
open_items.append(f"review: {len(bad)} runs failed")
|
|
691
|
+
if review is not None and any(f["grade"] >= 1 for f in review["findings"]) and verify is None:
|
|
692
|
+
open_items.append("verify: not run")
|
|
693
|
+
if verify is not None:
|
|
694
|
+
bad = [r["run"] for r in verify_recs if r.get("error") or r.get("verdicts") is None]
|
|
695
|
+
if bad:
|
|
696
|
+
open_items.append(f"verify: {len(bad)} runs have no verdicts")
|
|
697
|
+
if confirmed:
|
|
698
|
+
unfixed = [b["id"] for b in confirmed if b["id"] not in fixed]
|
|
699
|
+
if unfixed:
|
|
700
|
+
open_items.append(f"fix: {len(unfixed)} confirmed bugs have no accepted fix with a PASS judge")
|
|
701
|
+
return {"command": "status", "ok": True, "repo": audit["repo"], "commit": audit["commit"],
|
|
702
|
+
"targets": len(targets or []), "grades": counts, "confirmed_bugs": len(confirmed or []), "steps": steps,
|
|
703
|
+
"not_done": open_items, "cost_note": COST_NOTE}
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
# ---------------------------------------------------------------- gate
|
|
707
|
+
|
|
708
|
+
def cmd_waive(ctx: Context, bug: str, reason: str, by: Optional[str]) -> Dict[str, Any]:
|
|
709
|
+
"""Record the decision of the owner that one confirmed bug stays as it is."""
|
|
710
|
+
confirmed = read_json(ctx.out / "confirmed.json")
|
|
711
|
+
if confirmed is None:
|
|
712
|
+
raise UsageError("no confirmed.json: run `verify` first")
|
|
713
|
+
if bug not in {b["id"] for b in confirmed}:
|
|
714
|
+
raise UsageError(f"no confirmed bug with the id {bug}")
|
|
715
|
+
if not reason.strip():
|
|
716
|
+
raise UsageError("a waiver needs a reason")
|
|
717
|
+
waivers = read_json(ctx.out / "waivers.json", {})
|
|
718
|
+
waivers[bug] = {"reason": reason.strip(), "by": by, "time": now()}
|
|
719
|
+
write_json(ctx.out / "waivers.json", waivers)
|
|
720
|
+
return {"command": "waive", "ok": True, "bug": bug, "waivers": len(waivers)}
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def cmd_gate(ctx: Context, ref: Optional[str], repro_timeout: float) -> Dict[str, Any]:
|
|
724
|
+
"""Pass only when the audit is complete and each confirmed bug is closed. No model call.
|
|
725
|
+
|
|
726
|
+
With `--ref`, a bug is closed when each of its repros gives a new result on that commit.
|
|
727
|
+
With no `--ref`, a bug is closed when it has a fix with an accepted check and a PASS judge.
|
|
728
|
+
A waiver closes a bug in the two cases.
|
|
729
|
+
"""
|
|
730
|
+
audit = ctx.audit()
|
|
731
|
+
status = cmd_status(ctx)
|
|
732
|
+
incomplete = [x for x in status["not_done"] if not x.startswith("fix:")]
|
|
733
|
+
confirmed = read_json(ctx.out / "confirmed.json") or []
|
|
734
|
+
waivers = read_json(ctx.out / "waivers.json", {})
|
|
735
|
+
fix = read_json(ctx.out / "fix.json")
|
|
736
|
+
fixed = accepted_fixes([r for r in (read_json(ctx.record_path(i)) for i in (fix or {}).get("runs") or []) if r])
|
|
737
|
+
tree: Optional[Path] = None
|
|
738
|
+
commit: Optional[str] = None
|
|
739
|
+
if ref:
|
|
740
|
+
repo = Path(audit["repo"])
|
|
741
|
+
code, out = git(repo, "rev-parse", "--verify", f"{ref}^{{commit}}")
|
|
742
|
+
if code != 0:
|
|
743
|
+
raise UsageError(f"the ref {ref} is not a commit of {repo}")
|
|
744
|
+
commit = out.strip()
|
|
745
|
+
tree = ctx.out / "gate" / commit[:12]
|
|
746
|
+
if not tree.is_dir():
|
|
747
|
+
tree.mkdir(parents=True)
|
|
748
|
+
tarball = ctx.out / "gate.tar"
|
|
749
|
+
paths = [p for p in audit["paths"] if git(repo, "cat-file", "-e", f"{commit}:{p}")[0] == 0]
|
|
750
|
+
code, detail = git(repo, "archive", "-o", str(tarball), commit, *paths)
|
|
751
|
+
if code == 0:
|
|
752
|
+
code, detail = sh(["tar", "-x", "-f", str(tarball), "-C", str(tree)])
|
|
753
|
+
tarball.unlink(missing_ok=True)
|
|
754
|
+
if code != 0:
|
|
755
|
+
shutil.rmtree(tree, ignore_errors=True)
|
|
756
|
+
raise StepError(f"git archive failed: {detail[-300:]}")
|
|
757
|
+
rows: List[Dict[str, Any]] = []
|
|
758
|
+
for b in confirmed:
|
|
759
|
+
row = {"id": b["id"], "file": b["file"], "function": b["function"], "what_goes_wrong": b.get("what_goes_wrong"),
|
|
760
|
+
"severity": b.get("severity")}
|
|
761
|
+
if b["id"] in waivers:
|
|
762
|
+
row.update(state="waived", reason=waivers[b["id"]]["reason"])
|
|
763
|
+
elif tree is not None:
|
|
764
|
+
same, unknown = [], []
|
|
765
|
+
for rp in b["repros"]:
|
|
766
|
+
got = run_snippet(tree, Path(rp["path"]), ctx.home, timeout=repro_timeout)
|
|
767
|
+
if got.get("ok") and got.get("result") == rp["result_now"]:
|
|
768
|
+
same.append(rp["k"])
|
|
769
|
+
elif not got.get("ok") and (got.get("error") == "timeout" or got.get("error") in MISSING):
|
|
770
|
+
unknown.append(f"claim {rp['k']}: {got.get('error')}")
|
|
771
|
+
if same:
|
|
772
|
+
row.update(state="open", reason=f"the repro of claim {', '.join(str(k) for k in same)} gives the old result on {commit[:12]}")
|
|
773
|
+
elif unknown:
|
|
774
|
+
row.update(state="open", reason="the repro does not run on this commit (" + "; ".join(unknown) + "): fix the repro or record a waiver")
|
|
775
|
+
else:
|
|
776
|
+
row.update(state="closed", reason=f"each repro gives a new result on {commit[:12]}")
|
|
777
|
+
elif b["id"] in fixed:
|
|
778
|
+
row.update(state="closed", reason="a fix with an accepted check and a PASS judge is in its worktree; it is not merged by hpflow")
|
|
779
|
+
else:
|
|
780
|
+
row.update(state="open", reason="no accepted fix and no waiver")
|
|
781
|
+
rows.append(row)
|
|
782
|
+
open_rows = [r for r in rows if r["state"] == "open"]
|
|
783
|
+
ok = not incomplete and not open_rows
|
|
784
|
+
return {"command": "gate", "ok": ok, "verdict": "pass" if ok else "fail", "audited_commit": audit["commit"],
|
|
785
|
+
"checked_commit": commit, "targets": status["targets"], "confirmed_bugs": len(confirmed),
|
|
786
|
+
"closed": sum(1 for r in rows if r["state"] == "closed"), "waived": sum(1 for r in rows if r["state"] == "waived"),
|
|
787
|
+
"open": open_rows, "incomplete": incomplete, "bugs": rows,
|
|
788
|
+
"note": "A pass says that each bug that two verifier agents confirmed is closed or waived. It is not a proof that the code has no bug."}
|
|
789
|
+
|