patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,863 @@
|
|
|
1
|
+
"""Offline pipeline commands: ``python -m patchsight.pipeline <command>``."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import subprocess
|
|
6
|
+
import tarfile
|
|
7
|
+
import zipfile
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Annotated
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
import typer
|
|
14
|
+
|
|
15
|
+
from patchsight import db
|
|
16
|
+
from patchsight.eval.metrics import SetScores, score
|
|
17
|
+
from patchsight.pipeline import build_db, fix_sample, week1
|
|
18
|
+
from patchsight.pipeline.benchmarks import (
|
|
19
|
+
bundle,
|
|
20
|
+
changedfuncs,
|
|
21
|
+
dunlap,
|
|
22
|
+
extraction,
|
|
23
|
+
replay,
|
|
24
|
+
splits,
|
|
25
|
+
)
|
|
26
|
+
from patchsight.pipeline.diffs.github import GitHubClient
|
|
27
|
+
from patchsight.pipeline.extract.prompt import load_template
|
|
28
|
+
from patchsight.pipeline.ingest import ghsa_repo, snapshot
|
|
29
|
+
from patchsight.pipeline.ingest.aliases import alias_groups
|
|
30
|
+
from patchsight.pipeline.ingest.osv import Advisory, Excluded, Quarantined, normalise_dump
|
|
31
|
+
from patchsight.pipeline.llm.client import QuotaExhaustedError, RateLimitedError, client_for
|
|
32
|
+
from patchsight.pipeline.llm.config import load_config
|
|
33
|
+
from patchsight.pipeline.llm.tokens import HfTokenizer
|
|
34
|
+
from patchsight.pipeline.pypi import PyPIClient
|
|
35
|
+
|
|
36
|
+
app = typer.Typer(name="patchsight-pipeline", no_args_is_help=True, add_completion=False)
|
|
37
|
+
|
|
38
|
+
Root = Annotated[Path, typer.Option(help="Repository root.")]
|
|
39
|
+
DUMP = "osv-pypi-all"
|
|
40
|
+
MODIFIED = "osv-pypi-modified-id"
|
|
41
|
+
GHSA = "github-advisory-database"
|
|
42
|
+
MANIFEST = Path("manifests/snapshot.json")
|
|
43
|
+
RAW = Path("data/raw")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _load_verified(root: Path) -> tuple[snapshot.SnapshotManifest, Path]:
|
|
47
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
48
|
+
return manifest, snapshot.verify(manifest.file(DUMP), root)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command("snapshot")
|
|
52
|
+
def snapshot_command(root: Root = Path()) -> None:
|
|
53
|
+
"""Download the OSV PyPI dump and modified index, and pin their hashes."""
|
|
54
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
55
|
+
timeout = httpx.Timeout(60.0, connect=15.0)
|
|
56
|
+
with httpx.Client(timeout=timeout, follow_redirects=True) as client:
|
|
57
|
+
for name, url, filename in (
|
|
58
|
+
(DUMP, snapshot.OSV_PYPI_DUMP_URL, "all.zip"),
|
|
59
|
+
(MODIFIED, snapshot.OSV_PYPI_MODIFIED_URL, "modified_id.csv"),
|
|
60
|
+
):
|
|
61
|
+
entry = snapshot.download(client, name, url, root / RAW / "osv-pypi" / filename, root)
|
|
62
|
+
manifest = manifest.with_file(entry)
|
|
63
|
+
typer.echo(f"{name}: {entry.size:,} bytes, sha256 {entry.sha256}")
|
|
64
|
+
snapshot.save_manifest(manifest, root / MANIFEST)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@app.command("clone-ghsa")
|
|
68
|
+
def clone_ghsa_command(root: Root = Path()) -> None:
|
|
69
|
+
"""Sparse-clone the PyPI subset of github/advisory-database and pin its commit."""
|
|
70
|
+
manifest, dump = _load_verified(root)
|
|
71
|
+
advisories = [r for r in normalise_dump(dump) if isinstance(r, Advisory)]
|
|
72
|
+
dest = root / RAW / "advisory-database"
|
|
73
|
+
ghsa_repo.sparse_clone(dest, advisories)
|
|
74
|
+
entry = snapshot.record_repo(
|
|
75
|
+
GHSA, ghsa_repo.GHSA_REPO_URL, dest, root, note="sparse: PyPI GHSA advisories only"
|
|
76
|
+
)
|
|
77
|
+
snapshot.save_manifest(manifest.with_repo(entry), root / MANIFEST)
|
|
78
|
+
typer.echo(f"{GHSA}: {entry.commit}")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@app.command("normalise")
|
|
82
|
+
def normalise_command(
|
|
83
|
+
root: Root = Path(),
|
|
84
|
+
out: Annotated[Path, typer.Option(help="Output directory.")] = Path("data/interim"),
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Normalise the pinned dump into advisories, exclusions and quarantine (JSONL)."""
|
|
87
|
+
_, dump = _load_verified(root)
|
|
88
|
+
target = root / out
|
|
89
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
90
|
+
files = {
|
|
91
|
+
Advisory: (target / "advisories.jsonl").open("w", encoding="utf-8"),
|
|
92
|
+
Excluded: (target / "excluded.jsonl").open("w", encoding="utf-8"),
|
|
93
|
+
Quarantined: (target / "quarantine.jsonl").open("w", encoding="utf-8"),
|
|
94
|
+
}
|
|
95
|
+
counts = dict.fromkeys(files, 0)
|
|
96
|
+
try:
|
|
97
|
+
for result in normalise_dump(dump):
|
|
98
|
+
files[type(result)].write(result.model_dump_json() + "\n")
|
|
99
|
+
counts[type(result)] += 1
|
|
100
|
+
finally:
|
|
101
|
+
for handle in files.values():
|
|
102
|
+
handle.close()
|
|
103
|
+
typer.echo(", ".join(f"{kind.__name__}: {n:,}" for kind, n in counts.items()))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@app.command("week1-report")
|
|
107
|
+
def week1_report_command(
|
|
108
|
+
root: Root = Path(),
|
|
109
|
+
out: Annotated[Path, typer.Option(help="Report path, without extension.")] = Path(
|
|
110
|
+
"docs/reports/week1-data-gate"
|
|
111
|
+
),
|
|
112
|
+
) -> None:
|
|
113
|
+
"""Measure the Week 1 data gate and write the report as Markdown and JSON."""
|
|
114
|
+
manifest, dump = _load_verified(root)
|
|
115
|
+
results = list(normalise_dump(dump))
|
|
116
|
+
dump_entry = manifest.file(DUMP)
|
|
117
|
+
|
|
118
|
+
ghsa_scan = None
|
|
119
|
+
lines = [
|
|
120
|
+
f"- `{dump_entry.name}`: {dump_entry.url}, SHA-256 `{dump_entry.sha256}`, "
|
|
121
|
+
f"retrieved {dump_entry.retrieved_at:%Y-%m-%d %H:%M} UTC "
|
|
122
|
+
f"(Last-Modified: {dump_entry.last_modified})"
|
|
123
|
+
]
|
|
124
|
+
for repo in manifest.repos:
|
|
125
|
+
if repo.name == GHSA and (root / repo.path / ".git").exists():
|
|
126
|
+
ghsa_scan = ghsa_repo.scan(
|
|
127
|
+
root / repo.path, [r for r in results if isinstance(r, Advisory)]
|
|
128
|
+
)
|
|
129
|
+
lines.append(
|
|
130
|
+
f"- `{repo.name}`: {repo.url}, commit `{repo.commit}` "
|
|
131
|
+
f"({repo.committed_at:%Y-%m-%d}; {repo.note})"
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
report = week1.build_report(results, as_of=dump_entry.retrieved_at, ghsa_scan=ghsa_scan)
|
|
135
|
+
target = root / out
|
|
136
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
137
|
+
target.with_name(target.name + ".json").write_text(
|
|
138
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
139
|
+
)
|
|
140
|
+
target.with_name(target.name + ".md").write_text(week1.render_markdown(report, lines), "utf-8")
|
|
141
|
+
verdict = "PASS" if report.gate_passed else "FAIL"
|
|
142
|
+
typer.echo(f"gate {verdict}: {report.gate_metric.share:.1%} with a GitHub fix link")
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@app.command("fix-sample")
|
|
146
|
+
def fix_sample_command(
|
|
147
|
+
root: Root = Path(),
|
|
148
|
+
size: Annotated[int, typer.Option(help="Vulnerabilities to sample.")] = 100,
|
|
149
|
+
seed: Annotated[int, typer.Option(help="Random seed, recorded in the report.")] = 20261114,
|
|
150
|
+
cache: Annotated[Path, typer.Option(help="HTTP response cache.")] = Path(".cache/http"),
|
|
151
|
+
out: Annotated[Path, typer.Option(help="Report path, without extension.")] = Path(
|
|
152
|
+
"docs/reports/fix-candidate-sample"
|
|
153
|
+
),
|
|
154
|
+
) -> None:
|
|
155
|
+
"""Check a random sample of fix candidates against GitHub (needs GITHUB_TOKEN)."""
|
|
156
|
+
token = os.environ.get("GITHUB_TOKEN")
|
|
157
|
+
if not token:
|
|
158
|
+
raise typer.BadParameter("set GITHUB_TOKEN; unauthenticated GitHub allows 60 requests/h")
|
|
159
|
+
manifest, dump = _load_verified(root)
|
|
160
|
+
advisories = [r for r in normalise_dump(dump) if isinstance(r, Advisory)]
|
|
161
|
+
by_id = {a.id: a for a in advisories}
|
|
162
|
+
groups = alias_groups(advisories)
|
|
163
|
+
population = [
|
|
164
|
+
g
|
|
165
|
+
for g in groups
|
|
166
|
+
if any(
|
|
167
|
+
c.kind in week1.GITHUB_KINDS for i in g.advisory_ids for c in by_id[i].fix_candidates
|
|
168
|
+
)
|
|
169
|
+
]
|
|
170
|
+
sample = fix_sample.draw_sample(population, size, seed)
|
|
171
|
+
|
|
172
|
+
github = GitHubClient(root / cache / "github", token=token)
|
|
173
|
+
pypi = PyPIClient(root / cache / "pypi")
|
|
174
|
+
try:
|
|
175
|
+
checks = []
|
|
176
|
+
for number, group in enumerate(sample, 1):
|
|
177
|
+
checks.append(fix_sample.check_group(group, by_id, github, pypi))
|
|
178
|
+
if number % 10 == 0:
|
|
179
|
+
typer.echo(f"checked {number}/{len(sample)}", err=True)
|
|
180
|
+
finally:
|
|
181
|
+
github.close()
|
|
182
|
+
pypi.close()
|
|
183
|
+
|
|
184
|
+
dump_entry = manifest.file(DUMP)
|
|
185
|
+
report = fix_sample.build_report(
|
|
186
|
+
checks,
|
|
187
|
+
as_of=dump_entry.retrieved_at.date(),
|
|
188
|
+
seed=seed,
|
|
189
|
+
vulnerabilities=len(groups),
|
|
190
|
+
population=len(population),
|
|
191
|
+
)
|
|
192
|
+
lines = [
|
|
193
|
+
f"- `{dump_entry.name}`: SHA-256 `{dump_entry.sha256}` (see `manifests/snapshot.json`)"
|
|
194
|
+
]
|
|
195
|
+
target = root / out
|
|
196
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
197
|
+
target.with_name(target.name + ".json").write_text(
|
|
198
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
199
|
+
)
|
|
200
|
+
target.with_name(target.name + ".md").write_text(
|
|
201
|
+
fix_sample.render_markdown(report, lines), "utf-8"
|
|
202
|
+
)
|
|
203
|
+
typer.echo(
|
|
204
|
+
f"{report.verdict}: strict estimate {report.estimated_share_strict.share:.1%}, "
|
|
205
|
+
f"lenient {report.estimated_share_lenient.share:.1%} "
|
|
206
|
+
f"({github.http.network_requests} GitHub and {pypi.http.network_requests} PyPI requests)"
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
DUNLAP = "dunlap-llm-vulnerable-functions"
|
|
211
|
+
DUNLAP_DATA = "dunlap-patchparser-csv"
|
|
212
|
+
DUNLAP_ADVISORIES = "dunlap-govulndb-advisories-csv"
|
|
213
|
+
DUNLAP_SPLIT = Path("splits/dunlap-v1.json")
|
|
214
|
+
DUNLAP_BUNDLE = Path("benchmarks/dunlap-v1/candidates.json")
|
|
215
|
+
BASELINE_REPORT = Path("docs/reports/changedfuncs-baseline.json")
|
|
216
|
+
DUNLAP_DEV_SIZE = 60
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _github_token() -> str:
|
|
220
|
+
token = os.environ.get("GITHUB_TOKEN")
|
|
221
|
+
if not token:
|
|
222
|
+
raise typer.BadParameter("set GITHUB_TOKEN; unauthenticated GitHub allows 60 requests/h")
|
|
223
|
+
return token
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
@app.command("fetch-dunlap")
|
|
227
|
+
def fetch_dunlap_command(root: Root = Path()) -> None:
|
|
228
|
+
"""Clone the Dunlap et al. dataset and extract its CSVs.
|
|
229
|
+
|
|
230
|
+
If the manifest already pins the dataset, the pinned commit is checked out and both CSVs
|
|
231
|
+
are verified against their pinned hashes; nothing in the manifest changes. Only the
|
|
232
|
+
first run records new pins.
|
|
233
|
+
"""
|
|
234
|
+
dest = root / RAW / "llm-vulnerable-functions"
|
|
235
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
236
|
+
pinned = next((r for r in manifest.repos if r.name == DUNLAP), None)
|
|
237
|
+
if not (dest / ".git").exists():
|
|
238
|
+
ghsa_repo._git("clone", "-q", dunlap.DUNLAP_REPO_URL, str(dest))
|
|
239
|
+
if pinned is not None:
|
|
240
|
+
ghsa_repo._git("-C", str(dest), "checkout", "-q", pinned.commit)
|
|
241
|
+
target = dest / "data" / "patchparser-data"
|
|
242
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
243
|
+
with tarfile.open(dest / "data" / "patchparser-data.tar.gz") as archive:
|
|
244
|
+
archive.extractall(target, filter="data")
|
|
245
|
+
|
|
246
|
+
if pinned is not None:
|
|
247
|
+
for name in (DUNLAP_DATA, DUNLAP_ADVISORIES):
|
|
248
|
+
snapshot.verify(manifest.file(name), root)
|
|
249
|
+
typer.echo(f"{DUNLAP}: pinned commit {pinned.commit} checked out; CSV hashes verified")
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
repo = snapshot.record_repo(DUNLAP, dunlap.DUNLAP_REPO_URL, dest, root, note="Apache-2.0")
|
|
253
|
+
base = f"{dunlap.DUNLAP_REPO_URL.removesuffix('.git')}/blob/{repo.commit}/"
|
|
254
|
+
entries = []
|
|
255
|
+
for name, relative, published in (
|
|
256
|
+
(DUNLAP_DATA, dunlap.DUNLAP_CSV, "data/patchparser-data.tar.gz"),
|
|
257
|
+
(DUNLAP_ADVISORIES, extraction.DUNLAP_ADVISORIES, extraction.DUNLAP_ADVISORIES),
|
|
258
|
+
):
|
|
259
|
+
path = dest / relative
|
|
260
|
+
entries.append(
|
|
261
|
+
snapshot.SourceFile(
|
|
262
|
+
name=name,
|
|
263
|
+
url=base + published,
|
|
264
|
+
path=path.relative_to(root).as_posix(),
|
|
265
|
+
sha256=snapshot.sha256_file(path),
|
|
266
|
+
size=path.stat().st_size,
|
|
267
|
+
last_modified=None,
|
|
268
|
+
retrieved_at=datetime.now(UTC),
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
manifest = manifest.with_repo(repo)
|
|
272
|
+
for entry in entries:
|
|
273
|
+
manifest = manifest.with_file(entry)
|
|
274
|
+
snapshot.save_manifest(manifest, root / MANIFEST)
|
|
275
|
+
typer.echo(f"{DUNLAP}: pinned {repo.commit}")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@app.command("dunlap-baseline")
|
|
279
|
+
def dunlap_baseline_command(
|
|
280
|
+
root: Root = Path(),
|
|
281
|
+
cache: Annotated[Path, typer.Option(help="HTTP response cache.")] = Path(".cache/http"),
|
|
282
|
+
out: Annotated[Path, typer.Option(help="Report path, without extension.")] = Path(
|
|
283
|
+
"docs/reports/changedfuncs-baseline"
|
|
284
|
+
),
|
|
285
|
+
) -> None:
|
|
286
|
+
"""Reproduce ChangedFuncs on the Dunlap benchmark (needs GITHUB_TOKEN)."""
|
|
287
|
+
token = _github_token()
|
|
288
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
289
|
+
entry = manifest.file(DUNLAP_DATA)
|
|
290
|
+
reports = dunlap.load(snapshot.verify(entry, root))
|
|
291
|
+
repo = next(r for r in manifest.repos if r.name == DUNLAP)
|
|
292
|
+
github = GitHubClient(root / cache / "github", token=token)
|
|
293
|
+
try:
|
|
294
|
+
report = changedfuncs.run(github, reports, as_of=entry.retrieved_at.date())
|
|
295
|
+
finally:
|
|
296
|
+
github.close()
|
|
297
|
+
lines = [
|
|
298
|
+
f"- `{repo.name}`: {repo.url}, commit `{repo.commit}`",
|
|
299
|
+
f"- `{entry.name}`: `{dunlap.DUNLAP_CSV}`, SHA-256 `{entry.sha256}`",
|
|
300
|
+
"- Fix commits and file contents: GitHub REST API (Go's own repositories through "
|
|
301
|
+
"their `github.com/golang` mirrors)",
|
|
302
|
+
]
|
|
303
|
+
target = root / out
|
|
304
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
305
|
+
target.with_name(target.name + ".json").write_text(
|
|
306
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
307
|
+
)
|
|
308
|
+
target.with_name(target.name + ".md").write_text(
|
|
309
|
+
changedfuncs.render_markdown(report, lines), "utf-8"
|
|
310
|
+
)
|
|
311
|
+
o = report.ours
|
|
312
|
+
typer.echo(
|
|
313
|
+
f"ours: P {o.precision:.3f} R {o.recall:.3f} F1 {o.f1:.3f}; "
|
|
314
|
+
f"identical to released {report.identical_reports}/{report.reports}; "
|
|
315
|
+
f"{github.http.network_requests} GitHub requests"
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@app.command("dunlap-extract")
|
|
320
|
+
def dunlap_extract_command(
|
|
321
|
+
root: Root = Path(),
|
|
322
|
+
role: Annotated[str, typer.Option(help="Model role in the LLM config.")] = "extractor",
|
|
323
|
+
split: Annotated[str, typer.Option(help="Split part: dev, test or all.")] = "dev",
|
|
324
|
+
sample: Annotated[int | None, typer.Option(help="Run on a seeded sample of N reports.")] = None,
|
|
325
|
+
seed: Annotated[int, typer.Option(help="Sample seed.")] = extraction.SEED,
|
|
326
|
+
config: Annotated[Path, typer.Option(help="LLM config.")] = Path("configs/llm.toml"),
|
|
327
|
+
prompt: Annotated[Path, typer.Option(help="Prompt template.")] = Path("prompts/extract_v1.md"),
|
|
328
|
+
cache: Annotated[Path, typer.Option(help="Response cache root.")] = Path(".cache"),
|
|
329
|
+
out: Annotated[Path, typer.Option(help="Report path, without extension.")] = Path(
|
|
330
|
+
"docs/reports/extraction-dunlap"
|
|
331
|
+
),
|
|
332
|
+
) -> None:
|
|
333
|
+
"""Run LLM extraction on the Dunlap benchmark (needs GITHUB_TOKEN and the model's key)."""
|
|
334
|
+
token = _github_token()
|
|
335
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
336
|
+
data = manifest.file(DUNLAP_DATA)
|
|
337
|
+
reports = dunlap.load(snapshot.verify(data, root))
|
|
338
|
+
advisories = extraction.load_advisories(snapshot.verify(manifest.file(DUNLAP_ADVISORIES), root))
|
|
339
|
+
frozen = splits.load_split(root / DUNLAP_SPLIT)
|
|
340
|
+
llm_config = load_config(root / config)
|
|
341
|
+
template = load_template(root / prompt)
|
|
342
|
+
counter = HfTokenizer(llm_config.tokenizer, root / cache / "tokenizer")
|
|
343
|
+
llm, model_id = client_for(llm_config, role, root / cache / "llm", counter)
|
|
344
|
+
github = GitHubClient(root / cache / "http" / "github", token=token)
|
|
345
|
+
try:
|
|
346
|
+
result = extraction.run(
|
|
347
|
+
github, llm, llm_config, role, template, counter, reports, advisories,
|
|
348
|
+
as_of=data.retrieved_at.date(), sample_size=sample, seed=seed,
|
|
349
|
+
ids=frozen.split.ids(split), split=f"{frozen.split.name}:{split}",
|
|
350
|
+
split_sha256=frozen.sha256,
|
|
351
|
+
progress=lambda done, total: typer.echo(f"extracted {done}/{total}", err=True),
|
|
352
|
+
) # fmt: skip
|
|
353
|
+
except (QuotaExhaustedError, RateLimitedError) as stopped:
|
|
354
|
+
typer.echo(f"stopped: {stopped}", err=True)
|
|
355
|
+
typer.echo(
|
|
356
|
+
"Every answer so far is cached. Re-run the same command later to resume.", err=True
|
|
357
|
+
)
|
|
358
|
+
raise typer.Exit(code=2) from None
|
|
359
|
+
finally:
|
|
360
|
+
github.close()
|
|
361
|
+
lines = [
|
|
362
|
+
f"- Benchmark: `{DUNLAP_DATA}` SHA-256 `{data.sha256}` (see `manifests/snapshot.json`)",
|
|
363
|
+
f"- Prompt: `{prompt.as_posix()}` (`{template.version}`); config: `{config.as_posix()}`",
|
|
364
|
+
f"- Model: `{model_id}` via role `{role}`",
|
|
365
|
+
]
|
|
366
|
+
target = root / out
|
|
367
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
368
|
+
target.with_name(target.name + ".json").write_text(
|
|
369
|
+
result.model_dump_json(indent=2) + "\n", "utf-8"
|
|
370
|
+
)
|
|
371
|
+
target.with_name(target.name + ".md").write_text(
|
|
372
|
+
extraction.render_markdown(result, lines), "utf-8"
|
|
373
|
+
)
|
|
374
|
+
e, n = result.extraction, result.changedfuncs
|
|
375
|
+
typer.echo(
|
|
376
|
+
f"{result.reports} reports: extraction P {e.precision:.3f} R {e.recall:.3f} F1 {e.f1:.3f}; "
|
|
377
|
+
f"ChangedFuncs P {n.precision:.3f} R {n.recall:.3f} F1 {n.f1:.3f}; "
|
|
378
|
+
f"status {result.status}; {llm.network_calls} model calls"
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
@app.command("dunlap-split")
|
|
383
|
+
def dunlap_split_command(root: Root = Path()) -> None:
|
|
384
|
+
"""Freeze the Dunlap dev/test split once. Refuses to overwrite an existing split."""
|
|
385
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
386
|
+
reports = dunlap.load(snapshot.verify(manifest.file(DUNLAP_DATA), root))
|
|
387
|
+
split = splits.make_split(reports, "dunlap-v1", DUNLAP_DEV_SIZE, extraction.SEED)
|
|
388
|
+
record = splits.save_split(split, root / DUNLAP_SPLIT)
|
|
389
|
+
typer.echo(
|
|
390
|
+
f"{DUNLAP_SPLIT}: dev {len(split.dev)}, test {len(split.test)}, sha256 {record.sha256}"
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
KAGGLE_RUN_PREFIXES = (".cache/llm/", "docs/reports/", "run-info.json", "llama-server.log")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@app.command("import-kaggle-run")
|
|
398
|
+
def import_kaggle_run_command(
|
|
399
|
+
archive: Annotated[Path, typer.Argument(help="The zip downloaded from the Kaggle run.")],
|
|
400
|
+
root: Root = Path(),
|
|
401
|
+
) -> None:
|
|
402
|
+
"""Import a Kaggle run: model responses into the cache, its files into data/interim.
|
|
403
|
+
|
|
404
|
+
Only expected paths are accepted; anything absolute, containing "..", or outside the
|
|
405
|
+
expected prefixes is refused. Regenerate the reports afterwards with dunlap-extract
|
|
406
|
+
--role kaggle_extractor, which then replays from the cache with no model calls.
|
|
407
|
+
"""
|
|
408
|
+
inbox = root / "data" / "interim" / "kaggle" / archive.stem
|
|
409
|
+
with zipfile.ZipFile(archive) as bundle:
|
|
410
|
+
members = [m for m in bundle.infolist() if not m.is_dir()]
|
|
411
|
+
for member in members:
|
|
412
|
+
name = member.filename
|
|
413
|
+
if (
|
|
414
|
+
name.startswith("/")
|
|
415
|
+
or ".." in Path(name).parts
|
|
416
|
+
or not name.startswith(KAGGLE_RUN_PREFIXES)
|
|
417
|
+
):
|
|
418
|
+
raise typer.BadParameter(f"unexpected path in archive: {name!r}")
|
|
419
|
+
responses = 0
|
|
420
|
+
for member in members:
|
|
421
|
+
# Responses go into the shared cache; the run's usage ledger does not, so it can
|
|
422
|
+
# never overwrite this machine's daily-quota ledger.
|
|
423
|
+
if member.filename.startswith(".cache/llm/") and not member.filename.startswith(
|
|
424
|
+
".cache/llm/usage/"
|
|
425
|
+
):
|
|
426
|
+
target = root / member.filename
|
|
427
|
+
responses += member.filename.endswith(".json")
|
|
428
|
+
else:
|
|
429
|
+
target = inbox / member.filename
|
|
430
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
431
|
+
target.write_bytes(bundle.read(member))
|
|
432
|
+
typer.echo(f"imported {responses} cached responses; run files in {inbox.relative_to(root)}")
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
@app.command("build-db")
|
|
436
|
+
def build_db_command(
|
|
437
|
+
root: Root = Path(),
|
|
438
|
+
ids: Annotated[str | None, typer.Option(help="Comma-separated advisory IDs.")] = None,
|
|
439
|
+
sample: Annotated[int | None, typer.Option(help="Seeded sample of N vulnerabilities.")] = None,
|
|
440
|
+
seed: Annotated[int, typer.Option(help="Sample seed.")] = 20261114,
|
|
441
|
+
cache: Annotated[Path, typer.Option(help="HTTP response cache.")] = Path(".cache/http"),
|
|
442
|
+
out: Annotated[Path, typer.Option(help="Output path, without extension.")] = Path(
|
|
443
|
+
"data/build/patchsight"
|
|
444
|
+
),
|
|
445
|
+
) -> None:
|
|
446
|
+
"""Build a symbol database of fallback records (needs GITHUB_TOKEN)."""
|
|
447
|
+
token = _github_token()
|
|
448
|
+
manifest, dump = _load_verified(root)
|
|
449
|
+
advisories = [r for r in normalise_dump(dump) if isinstance(r, Advisory)]
|
|
450
|
+
by_id = {a.id: a for a in advisories}
|
|
451
|
+
groups = [
|
|
452
|
+
g
|
|
453
|
+
for g in alias_groups(advisories)
|
|
454
|
+
if any(
|
|
455
|
+
c.kind in week1.GITHUB_KINDS for i in g.advisory_ids for c in by_id[i].fix_candidates
|
|
456
|
+
)
|
|
457
|
+
]
|
|
458
|
+
if ids:
|
|
459
|
+
wanted = {i.strip() for i in ids.split(",") if i.strip()}
|
|
460
|
+
groups = [g for g in groups if wanted & set(g.identifiers)]
|
|
461
|
+
elif sample is not None:
|
|
462
|
+
groups = fix_sample.draw_sample(groups, sample, seed)
|
|
463
|
+
dump_entry = manifest.file(DUMP)
|
|
464
|
+
version = dump_entry.retrieved_at.strftime("%Y.%m.%d")
|
|
465
|
+
# A full build takes hours: wait out GitHub's hourly limit instead of failing.
|
|
466
|
+
github = GitHubClient(root / cache / "github", token=token, wait_for_rate_limit=3900)
|
|
467
|
+
pypi = PyPIClient(root / cache / "pypi")
|
|
468
|
+
try:
|
|
469
|
+
records, aliases, report = build_db.build(
|
|
470
|
+
groups, by_id, github, pypi, version,
|
|
471
|
+
progress=lambda done, total: typer.echo(f"built {done}/{total}", err=True),
|
|
472
|
+
) # fmt: skip
|
|
473
|
+
finally:
|
|
474
|
+
github.close()
|
|
475
|
+
pypi.close()
|
|
476
|
+
commit = subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
477
|
+
["git", "-C", str(root), "rev-parse", "HEAD"], # noqa: S607
|
|
478
|
+
capture_output=True, text=True, check=False,
|
|
479
|
+
).stdout.strip() # fmt: skip
|
|
480
|
+
meta = {
|
|
481
|
+
"db_version": version,
|
|
482
|
+
"built_at": datetime.now(UTC).isoformat(),
|
|
483
|
+
"builder": "fallback-v1",
|
|
484
|
+
"snapshot_sha256": dump_entry.sha256,
|
|
485
|
+
"code_commit": commit,
|
|
486
|
+
}
|
|
487
|
+
target = root / out
|
|
488
|
+
db.write(target.with_name(target.name + ".sqlite"), records, aliases, meta)
|
|
489
|
+
target.with_name(target.name + ".jsonl").write_text(build_db.jsonl(records), "utf-8")
|
|
490
|
+
target.with_name(target.name + "-build.json").write_text(
|
|
491
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
492
|
+
)
|
|
493
|
+
typer.echo(
|
|
494
|
+
f"db {version}: {report.records} records, {report.symbols} symbols from "
|
|
495
|
+
f"{report.groups} vulnerabilities; {report.status}"
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
@app.command("export-dunlap-bundle")
|
|
500
|
+
def export_dunlap_bundle_command(
|
|
501
|
+
root: Root = Path(),
|
|
502
|
+
cache: Annotated[Path, typer.Option(help="HTTP response cache.")] = Path(".cache/http"),
|
|
503
|
+
) -> None:
|
|
504
|
+
"""Write the benchmark bundle: candidate metadata and truth, no code (needs GITHUB_TOKEN)."""
|
|
505
|
+
token = _github_token()
|
|
506
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
507
|
+
data = manifest.file(DUNLAP_DATA)
|
|
508
|
+
repo = next(r for r in manifest.repos if r.name == DUNLAP)
|
|
509
|
+
reports = dunlap.load(snapshot.verify(data, root))
|
|
510
|
+
github = GitHubClient(root / cache / "github", token=token)
|
|
511
|
+
try:
|
|
512
|
+
exported = bundle.export(
|
|
513
|
+
github, reports, {"dataset_commit": repo.commit, "dataset_csv_sha256": data.sha256}
|
|
514
|
+
)
|
|
515
|
+
finally:
|
|
516
|
+
github.close()
|
|
517
|
+
bundle.save(exported, root / DUNLAP_BUNDLE)
|
|
518
|
+
count = sum(len(r.candidates) for r in exported.reports)
|
|
519
|
+
typer.echo(f"{DUNLAP_BUNDLE}: {len(exported.reports)} reports, {count} candidates")
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
@app.command("export-replay")
|
|
523
|
+
def export_replay_command(
|
|
524
|
+
tag: Annotated[str, typer.Argument(help="Report tag, e.g. extract-v1-qwen3.8-27b-q4km.")],
|
|
525
|
+
root: Root = Path(),
|
|
526
|
+
role: Annotated[str, typer.Option(help="Model role the run used.")] = "kaggle_extractor",
|
|
527
|
+
part: Annotated[str, typer.Option("--split", help="Split part the run covered.")] = "all",
|
|
528
|
+
config: Annotated[Path, typer.Option(help="LLM config.")] = Path("configs/llm.toml"),
|
|
529
|
+
prompt: Annotated[Path, typer.Option(help="Prompt template.")] = Path("prompts/extract_v1.md"),
|
|
530
|
+
cache: Annotated[Path, typer.Option(help="Cache root.")] = Path(".cache"),
|
|
531
|
+
) -> None:
|
|
532
|
+
"""Write a replay bundle from the response cache only (never calls a model)."""
|
|
533
|
+
token = _github_token()
|
|
534
|
+
manifest = snapshot.load_manifest(root / MANIFEST)
|
|
535
|
+
ids = splits.load_split(root / DUNLAP_SPLIT).split.ids(part)
|
|
536
|
+
reports = [
|
|
537
|
+
r for r in dunlap.load(snapshot.verify(manifest.file(DUNLAP_DATA), root)) if r.id in ids
|
|
538
|
+
]
|
|
539
|
+
advisories = extraction.load_advisories(snapshot.verify(manifest.file(DUNLAP_ADVISORIES), root))
|
|
540
|
+
llm_config = load_config(root / config)
|
|
541
|
+
counter = HfTokenizer(llm_config.tokenizer, root / cache / "tokenizer")
|
|
542
|
+
github = GitHubClient(root / cache / "http" / "github", token=token)
|
|
543
|
+
try:
|
|
544
|
+
exported = replay.export(
|
|
545
|
+
github, root / cache / "llm", llm_config, role, load_template(root / prompt), counter,
|
|
546
|
+
reports, advisories, split=part,
|
|
547
|
+
) # fmt: skip
|
|
548
|
+
finally:
|
|
549
|
+
github.close()
|
|
550
|
+
target = root / DUNLAP_BUNDLE.parent / f"extraction-{tag}.json"
|
|
551
|
+
replay.save(exported, target)
|
|
552
|
+
answers = sum(len(r.answers) for r in exported.reports)
|
|
553
|
+
typer.echo(f"{target.relative_to(root)}: {len(exported.reports)} reports, {answers} answers")
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
@app.command("confidence-report")
|
|
557
|
+
def confidence_report_command(
|
|
558
|
+
primary: Annotated[str, typer.Option(help="Extractor run tag (replay bundle).")],
|
|
559
|
+
secondary: Annotated[str, typer.Option(help="Second-opinion run tag (replay bundle).")],
|
|
560
|
+
part: Annotated[str, typer.Option("--split", help="Split part: dev, test or all.")] = "dev",
|
|
561
|
+
root: Root = Path(),
|
|
562
|
+
) -> None:
|
|
563
|
+
"""Compare confidence rules, in the benchmark and safety views, from committed files."""
|
|
564
|
+
from patchsight.eval import confidence
|
|
565
|
+
from patchsight.eval.safety import safety
|
|
566
|
+
|
|
567
|
+
data = bundle.load(root / DUNLAP_BUNDLE)
|
|
568
|
+
frozen = splits.load_split(root / DUNLAP_SPLIT)
|
|
569
|
+
ids = frozen.split.ids(part)
|
|
570
|
+
truth = {r.id: set(r.symbols) for r in data.reports if r.id in ids}
|
|
571
|
+
changed = {r.id: r.changed_functions() for r in data.reports if r.id in ids}
|
|
572
|
+
reports_dir = root / "docs" / "reports"
|
|
573
|
+
runs = {}
|
|
574
|
+
for tag in (primary, secondary):
|
|
575
|
+
run = replay.load(root / DUNLAP_BUNDLE.parent / f"extraction-{tag}.json")
|
|
576
|
+
# Validated entry points come from the run's committed report (any part that has them).
|
|
577
|
+
entry: dict[str, frozenset[str]] = {}
|
|
578
|
+
for source in sorted(reports_dir.glob(f"extraction-dunlap-*-{tag}.json")):
|
|
579
|
+
for outcome in extraction.ExtractionRun.model_validate_json(
|
|
580
|
+
source.read_text("utf-8")
|
|
581
|
+
).outcomes:
|
|
582
|
+
entry.setdefault(outcome.advisory_id, frozenset(outcome.entry_points))
|
|
583
|
+
answers = {}
|
|
584
|
+
for report in run.reports:
|
|
585
|
+
if report.id in ids:
|
|
586
|
+
status, names = replay.replay_report(report, run.prompt_version)
|
|
587
|
+
answers[report.id] = confidence.Answer(
|
|
588
|
+
status=status, vulnerable=frozenset(names), truncated=report.truncated,
|
|
589
|
+
entry_points=entry.get(report.id, frozenset()),
|
|
590
|
+
) # fmt: skip
|
|
591
|
+
runs[tag] = (run.model, answers)
|
|
592
|
+
first, second = runs[primary][1], runs[secondary][1]
|
|
593
|
+
results = [
|
|
594
|
+
confidence.evaluate(first, second, changed, truth, rule, True, require_public=public)
|
|
595
|
+
for public in (False, True)
|
|
596
|
+
for rule in confidence.RULES
|
|
597
|
+
]
|
|
598
|
+
alone = score({r: set(a.vulnerable) for r, a in first.items()}, truth)
|
|
599
|
+
naive = score(changed, truth)
|
|
600
|
+
naive_safety = safety(changed, truth)
|
|
601
|
+
alone_safety = safety(
|
|
602
|
+
{r: set(a.vulnerable) | set(a.entry_points) for r, a in first.items()}, truth
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
def strict(label: str, s: SetScores) -> str:
|
|
606
|
+
return f"| {label} | {s.precision:.3f} | {s.recall:.3f} | {s.f1:.3f} | {s.miss_rate:.1%} |"
|
|
607
|
+
|
|
608
|
+
def chosen(r: confidence.TierScores) -> str:
|
|
609
|
+
return " **(chosen)**" if r.rule == confidence.CHOSEN_RULE and r.require_public else ""
|
|
610
|
+
|
|
611
|
+
lines = [
|
|
612
|
+
f"# Confidence tiers on the Dunlap `{part}` split",
|
|
613
|
+
"",
|
|
614
|
+
"Generated by `python -m patchsight.pipeline confidence-report`; do not edit by hand. "
|
|
615
|
+
"Computed offline from committed replay bundles and reports.",
|
|
616
|
+
"",
|
|
617
|
+
f"- Extractor: `{runs[primary][0]}` (`{primary}`)",
|
|
618
|
+
f"- Second opinion: `{runs[secondary][0]}` (`{secondary}`)",
|
|
619
|
+
f"- Split `{frozen.split.name}:{part}` (SHA-256 `{frozen.sha256}`), {len(first)} reports",
|
|
620
|
+
"",
|
|
621
|
+
"Two views. The **benchmark view** compares function names exactly and ignores entry "
|
|
622
|
+
"points, so it stays comparable with Dunlap et al. The **safety view** scores what the "
|
|
623
|
+
"scanner matches: functions plus validated entry points, with names matched on a dotted "
|
|
624
|
+
"boundary (`Conn.decryptTicket` matches `decryptTicket`). Low confidence always falls "
|
|
625
|
+
"back to every changed function (plus entry points in the safety view).",
|
|
626
|
+
"",
|
|
627
|
+
"## Benchmark view",
|
|
628
|
+
"",
|
|
629
|
+
"| System | Precision | Recall | F1 | Miss rate |",
|
|
630
|
+
"|---|---|---|---|---|",
|
|
631
|
+
strict("ChangedFuncs", naive),
|
|
632
|
+
strict(f"Extractor alone ({runs[primary][0]})", alone),
|
|
633
|
+
]
|
|
634
|
+
lines += [
|
|
635
|
+
strict(
|
|
636
|
+
f"Tiers: `{r.rule}`{', public symbol required' if r.require_public else ''}{chosen(r)}",
|
|
637
|
+
r.overall,
|
|
638
|
+
)
|
|
639
|
+
for r in results
|
|
640
|
+
]
|
|
641
|
+
lines += [
|
|
642
|
+
"",
|
|
643
|
+
"## Safety view",
|
|
644
|
+
"",
|
|
645
|
+
"The high-confidence miss rate is the number the proposal targets at 2% or less: a miss "
|
|
646
|
+
'there is what could turn into a wrong "not called".',
|
|
647
|
+
"",
|
|
648
|
+
"| System | Recall | Miss rate | High-confidence reports | High-confidence miss rate |",
|
|
649
|
+
"|---|---|---|---|---|",
|
|
650
|
+
f"| ChangedFuncs | {naive_safety.recall:.3f} | {naive_safety.miss_rate:.1%} | none | n/a |",
|
|
651
|
+
f"| Extractor alone, with entry points | {alone_safety.recall:.3f} | "
|
|
652
|
+
f"{alone_safety.miss_rate:.1%} | none | n/a |",
|
|
653
|
+
]
|
|
654
|
+
for r in results:
|
|
655
|
+
missed = round(r.safety_high.miss_rate * r.safety_high.reports)
|
|
656
|
+
label = (
|
|
657
|
+
f"Tiers: `{r.rule}`{', public symbol required' if r.require_public else ''}{chosen(r)}"
|
|
658
|
+
)
|
|
659
|
+
lines.append(
|
|
660
|
+
f"| {label} | {r.safety_overall.recall:.3f} | {r.safety_overall.miss_rate:.1%} | "
|
|
661
|
+
f"{r.high_reports} ({r.high_share:.0%}) | {r.safety_high.miss_rate:.1%} "
|
|
662
|
+
f"({missed} of {r.safety_high.reports}) |"
|
|
663
|
+
)
|
|
664
|
+
lines += ["", "Why reports were low confidence:", ""]
|
|
665
|
+
for r in results:
|
|
666
|
+
reasons = ", ".join(f"{k} {v}" for k, v in r.low_reasons.items())
|
|
667
|
+
lines.append(f"- `{r.rule}`{', public' if r.require_public else ''}: {reasons}")
|
|
668
|
+
lines.append("")
|
|
669
|
+
target = reports_dir / f"confidence-dunlap-{part}-{primary}-vs-{secondary}"
|
|
670
|
+
target.with_name(target.name + ".md").write_text("\n".join(lines), "utf-8")
|
|
671
|
+
target.with_name(target.name + ".json").write_text(
|
|
672
|
+
json.dumps([r.model_dump(mode="json") for r in results], indent=2) + "\n", "utf-8"
|
|
673
|
+
)
|
|
674
|
+
typer.echo("\n".join(lines[10:]))
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
@app.command("parity-report")
|
|
678
|
+
def parity_report_command(
|
|
679
|
+
reference: Annotated[str, typer.Option(help="Reference run tag, e.g. the self-hosted run.")],
|
|
680
|
+
candidate: Annotated[str, typer.Option(help="Run tag compared against the reference.")],
|
|
681
|
+
part: Annotated[str, typer.Option("--split", help="Split part: dev, test or all.")] = "dev",
|
|
682
|
+
root: Root = Path(),
|
|
683
|
+
) -> None:
|
|
684
|
+
"""Compare two extraction runs of the same prompt, from their committed reports."""
|
|
685
|
+
from patchsight.eval.parity import VALID, parity
|
|
686
|
+
|
|
687
|
+
data = bundle.load(root / DUNLAP_BUNDLE)
|
|
688
|
+
frozen = splits.load_split(root / DUNLAP_SPLIT)
|
|
689
|
+
ids = frozen.split.ids(part)
|
|
690
|
+
truth = {r.id: set(r.symbols) for r in data.reports if r.id in ids}
|
|
691
|
+
reports_dir = root / "docs" / "reports"
|
|
692
|
+
runs: dict[str, extraction.ExtractionRun] = {}
|
|
693
|
+
for tag in (reference, candidate):
|
|
694
|
+
source = reports_dir / f"extraction-dunlap-{part}-{tag}.json"
|
|
695
|
+
run = extraction.ExtractionRun.model_validate_json(source.read_text("utf-8"))
|
|
696
|
+
if run.split_sha256 != frozen.sha256 or run.sample_size is not None:
|
|
697
|
+
raise typer.BadParameter(f"{source} is not a full run of the frozen split")
|
|
698
|
+
runs[tag] = run
|
|
699
|
+
if runs[reference].prompt_version != runs[candidate].prompt_version:
|
|
700
|
+
raise typer.BadParameter("the two runs used different prompt versions")
|
|
701
|
+
answers = {
|
|
702
|
+
tag: {
|
|
703
|
+
o.advisory_id: (set(o.vulnerable) if o.status in VALID else None) for o in run.outcomes
|
|
704
|
+
}
|
|
705
|
+
for tag, run in runs.items()
|
|
706
|
+
}
|
|
707
|
+
p = parity(answers[reference], answers[candidate], truth)
|
|
708
|
+
ref_model, cand_model = runs[reference].model, runs[candidate].model
|
|
709
|
+
|
|
710
|
+
def row(label: str, s: SetScores) -> str:
|
|
711
|
+
return f"| {label} | {s.precision:.3f} | {s.recall:.3f} | {s.f1:.3f} | {s.miss_rate:.1%} |"
|
|
712
|
+
|
|
713
|
+
def failed(names: tuple[str, ...]) -> str:
|
|
714
|
+
return ", ".join(f"`{n}`" for n in names) if names else "none"
|
|
715
|
+
|
|
716
|
+
lines = [
|
|
717
|
+
f"# Parity on the Dunlap `{part}` split: `{reference}` and `{candidate}`",
|
|
718
|
+
"",
|
|
719
|
+
"Generated by `python -m patchsight.pipeline parity-report`; do not edit by hand. "
|
|
720
|
+
"Computed offline from the two committed extraction reports.",
|
|
721
|
+
"",
|
|
722
|
+
f"- Reference: `{ref_model}` (`{reference}`)",
|
|
723
|
+
f"- Candidate: `{cand_model}` (`{candidate}`)",
|
|
724
|
+
f"- Prompt `{runs[reference].prompt_version}`; split `{frozen.split.name}:{part}` "
|
|
725
|
+
f"(SHA-256 `{frozen.sha256}`), {p.reports} reports",
|
|
726
|
+
"",
|
|
727
|
+
"## Reliability",
|
|
728
|
+
"",
|
|
729
|
+
f"- Reference without a valid answer ({len(p.reference_failed)}): "
|
|
730
|
+
f"{failed(p.reference_failed)}",
|
|
731
|
+
f"- Candidate without a valid answer ({len(p.candidate_failed)}): "
|
|
732
|
+
f"{failed(p.candidate_failed)}",
|
|
733
|
+
"",
|
|
734
|
+
f"## Quality on the {p.paired} reports both runs answered",
|
|
735
|
+
"",
|
|
736
|
+
"| Run | Precision | Recall | F1 | Miss rate |",
|
|
737
|
+
"|---|---|---|---|---|",
|
|
738
|
+
row(f"Reference ({ref_model})", p.reference),
|
|
739
|
+
row(f"Candidate ({cand_model})", p.candidate),
|
|
740
|
+
"",
|
|
741
|
+
f"F1 difference, reference minus candidate: **{p.f1_difference:+.3f}** (95% CI "
|
|
742
|
+
f"{p.f1_difference_ci95.low:+.3f} to {p.f1_difference_ci95.high:+.3f}; paired bootstrap "
|
|
743
|
+
"over reports).",
|
|
744
|
+
"",
|
|
745
|
+
"## Agreement",
|
|
746
|
+
"",
|
|
747
|
+
f"- Identical vulnerable sets: {p.identical} of {p.paired}.",
|
|
748
|
+
f"- Of the {len(p.differing)} that differ: the candidate only adds symbols in "
|
|
749
|
+
f"{p.candidate_adds_only}, only drops symbols in {p.candidate_drops_only}, and does both "
|
|
750
|
+
f"in {p.both_add_and_drop}.",
|
|
751
|
+
f"- Differing reports: {failed(p.differing)}",
|
|
752
|
+
"",
|
|
753
|
+
]
|
|
754
|
+
target = reports_dir / f"parity-dunlap-{part}-{reference}-vs-{candidate}"
|
|
755
|
+
target.with_name(target.name + ".md").write_text("\n".join(lines), "utf-8")
|
|
756
|
+
target.with_name(target.name + ".json").write_text(p.model_dump_json(indent=2) + "\n", "utf-8")
|
|
757
|
+
typer.echo("\n".join(lines[9:]))
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
@app.command("reproduce")
|
|
761
|
+
def reproduce_command(root: Root = Path()) -> None:
|
|
762
|
+
"""Recompute the benchmark tables offline from committed files. No network, no API key."""
|
|
763
|
+
data = bundle.load(root / DUNLAP_BUNDLE)
|
|
764
|
+
frozen = splits.load_split(root / DUNLAP_SPLIT)
|
|
765
|
+
lines = ["ChangedFuncs on the Dunlap Go benchmark (unit: report, function)", ""]
|
|
766
|
+
lines.append(f"{'part':<6} {'reports':>7} {'cand.':>6} {'TP':>4} {'P':>6} {'R':>6} {'F1':>6}")
|
|
767
|
+
results = {}
|
|
768
|
+
for part in ("all", "dev", "test"):
|
|
769
|
+
s = bundle.changedfuncs_scores(data, frozen.split.ids(part))
|
|
770
|
+
results[part] = s
|
|
771
|
+
lines.append(
|
|
772
|
+
f"{part:<6} {s.reports:>7} {s.true_positives + s.false_positives:>6} "
|
|
773
|
+
f"{s.true_positives:>4} {s.precision:>6.3f} {s.recall:>6.3f} {s.f1:>6.3f}"
|
|
774
|
+
)
|
|
775
|
+
lines += ["", "Paper (Dunlap et al., DIMVA 2024): P 0.22, R 0.90, F1 0.35 on 2,370 candidates"]
|
|
776
|
+
typer.echo("\n".join(lines))
|
|
777
|
+
|
|
778
|
+
committed = changedfuncs.BaselineReport.model_validate_json(
|
|
779
|
+
(root / BASELINE_REPORT).read_text(encoding="utf-8")
|
|
780
|
+
).ours
|
|
781
|
+
mine = results["all"]
|
|
782
|
+
same = (mine.true_positives, mine.false_positives, mine.false_negatives) == (
|
|
783
|
+
committed.true_positives, committed.false_positives, committed.false_negatives,
|
|
784
|
+
) # fmt: skip
|
|
785
|
+
if not same:
|
|
786
|
+
typer.echo(f"MISMATCH with {BASELINE_REPORT}", err=True)
|
|
787
|
+
raise typer.Exit(code=1)
|
|
788
|
+
typer.echo(f"\nMatches {BASELINE_REPORT} exactly.")
|
|
789
|
+
|
|
790
|
+
truth = {r.id: set(r.symbols) for r in data.reports}
|
|
791
|
+
for path in sorted((root / DUNLAP_BUNDLE.parent).glob("extraction-*.json")):
|
|
792
|
+
tag = path.stem.removeprefix("extraction-")
|
|
793
|
+
run = replay.load(path)
|
|
794
|
+
typer.echo(f"\nLLM extraction, {run.model}, {run.prompt_version} (replayed offline)\n")
|
|
795
|
+
typer.echo(f"{'part':<6} {'reports':>7} {'TP':>4} {'P':>6} {'R':>6} {'F1':>6} outcomes")
|
|
796
|
+
for part in ("all", "dev", "test"):
|
|
797
|
+
report_path = root / "docs" / "reports" / f"extraction-dunlap-{part}-{tag}.json"
|
|
798
|
+
if not report_path.exists():
|
|
799
|
+
continue # this run did not cover that part
|
|
800
|
+
s, statuses = replay.replay(run, truth, frozen.split.ids(part))
|
|
801
|
+
outcomes = ", ".join(f"{k} {v}" for k, v in statuses.items())
|
|
802
|
+
typer.echo(
|
|
803
|
+
f"{part:<6} {s.reports:>7} {s.true_positives:>4} {s.precision:>6.3f} "
|
|
804
|
+
f"{s.recall:>6.3f} {s.f1:>6.3f} {outcomes}"
|
|
805
|
+
)
|
|
806
|
+
published = extraction.ExtractionRun.model_validate_json(
|
|
807
|
+
report_path.read_text(encoding="utf-8")
|
|
808
|
+
)
|
|
809
|
+
expected = (published.extraction.true_positives, published.extraction.false_positives,
|
|
810
|
+
published.extraction.false_negatives, published.status) # fmt: skip
|
|
811
|
+
if (s.true_positives, s.false_positives, s.false_negatives, statuses) != expected:
|
|
812
|
+
typer.echo(f"MISMATCH with {report_path.relative_to(root)}", err=True)
|
|
813
|
+
raise typer.Exit(code=1)
|
|
814
|
+
typer.echo(f"\nMatches docs/reports/extraction-dunlap-*-{tag}.json exactly.")
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
@app.command("package-db")
|
|
818
|
+
def package_db_command(
|
|
819
|
+
database: Annotated[Path, typer.Argument(help="A database built by build-db.")],
|
|
820
|
+
root: Root = Path(),
|
|
821
|
+
repo: Annotated[str, typer.Option(help="GitHub repository the release lives in.")] = (
|
|
822
|
+
"professor3333/PatchSight"
|
|
823
|
+
),
|
|
824
|
+
pointer: Annotated[Path, typer.Option(help="Pointer file to write.")] = Path(
|
|
825
|
+
"releases/latest.json"
|
|
826
|
+
),
|
|
827
|
+
) -> None:
|
|
828
|
+
"""Compress a database for release and write the pointer the CLI downloads it by."""
|
|
829
|
+
import gzip
|
|
830
|
+
import shutil
|
|
831
|
+
|
|
832
|
+
opened = db.SymbolDatabase(database)
|
|
833
|
+
meta = dict(opened.meta)
|
|
834
|
+
records = opened.record_count()
|
|
835
|
+
opened.close()
|
|
836
|
+
version = meta["db_version"]
|
|
837
|
+
tag = f"db-{version}"
|
|
838
|
+
asset = root / "data" / "build" / "release" / f"patchsight-db-{version}.sqlite.gz"
|
|
839
|
+
asset.parent.mkdir(parents=True, exist_ok=True)
|
|
840
|
+
# mtime=0 keeps the archive byte-identical for the same database, so its hash is stable.
|
|
841
|
+
with (
|
|
842
|
+
database.open("rb") as source,
|
|
843
|
+
asset.open("wb") as raw,
|
|
844
|
+
gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as sink,
|
|
845
|
+
):
|
|
846
|
+
shutil.copyfileobj(source, sink)
|
|
847
|
+
record = {
|
|
848
|
+
"db_version": version,
|
|
849
|
+
"schema_version": int(meta["schema_version"]),
|
|
850
|
+
"url": f"https://github.com/{repo}/releases/download/{tag}/{asset.name}",
|
|
851
|
+
"sha256": snapshot.sha256_file(asset),
|
|
852
|
+
"size": asset.stat().st_size,
|
|
853
|
+
"built_at": meta["built_at"],
|
|
854
|
+
"records": records,
|
|
855
|
+
}
|
|
856
|
+
(root / pointer).parent.mkdir(parents=True, exist_ok=True)
|
|
857
|
+
(root / pointer).write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
|
|
858
|
+
typer.echo(f"{asset.relative_to(root)}: {record['size']:,} bytes, sha256 {record['sha256']}")
|
|
859
|
+
typer.echo(f"wrote {pointer}; publish with: gh release create {tag} {asset.relative_to(root)}")
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
if __name__ == "__main__":
|
|
863
|
+
app()
|