patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
"""Check a random sample of fix candidates against GitHub: are they really fixes?
|
|
2
|
+
|
|
3
|
+
This is the condition attached to passing the Week 1 gate. A link to a commit is only a
|
|
4
|
+
*candidate*. A vulnerability counts as having a real fix only when a candidate:
|
|
5
|
+
|
|
6
|
+
1. resolves on GitHub (a commit, a merged pull request or a tag comparison);
|
|
7
|
+
2. sits on the package's own repository, as declared in its PyPI metadata or an OSV
|
|
8
|
+
``GIT`` range (renames followed); and
|
|
9
|
+
3. changes at least one non-test source file.
|
|
10
|
+
|
|
11
|
+
These are necessary conditions checked by code, with no LLM and no hand labelling. They
|
|
12
|
+
do not prove a commit is *the* fix; whether the right functions changed is measured later
|
|
13
|
+
against labelled data.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import math
|
|
17
|
+
import random
|
|
18
|
+
import statistics
|
|
19
|
+
from collections import Counter
|
|
20
|
+
from collections.abc import Iterable, Sequence
|
|
21
|
+
from datetime import date
|
|
22
|
+
from typing import Literal
|
|
23
|
+
|
|
24
|
+
from pydantic import BaseModel, ConfigDict
|
|
25
|
+
|
|
26
|
+
from patchsight.pipeline.diffs.github import ChangeSet, GitHubClient
|
|
27
|
+
from patchsight.pipeline.diffs.paths import classify_path
|
|
28
|
+
from patchsight.pipeline.http import HttpError, NotFoundError, RateLimitError
|
|
29
|
+
from patchsight.pipeline.ingest.aliases import AliasGroup
|
|
30
|
+
from patchsight.pipeline.ingest.osv import Advisory
|
|
31
|
+
from patchsight.pipeline.ingest.references import FixCandidate
|
|
32
|
+
from patchsight.pipeline.pypi import PyPIClient
|
|
33
|
+
from patchsight.pipeline.week1 import GATE_THRESHOLD, GITHUB_KINDS, Share, wilson
|
|
34
|
+
|
|
35
|
+
CandidateStatus = Literal["resolved", "not_found", "unmerged", "error"]
|
|
36
|
+
Outcome = Literal[
|
|
37
|
+
"fix_on_package_repo",
|
|
38
|
+
"fix_repo_unconfirmed",
|
|
39
|
+
"non_source_only",
|
|
40
|
+
"other_repo_only",
|
|
41
|
+
"unmerged_pr_only",
|
|
42
|
+
"unresolvable",
|
|
43
|
+
]
|
|
44
|
+
OUTCOME_ORDER: tuple[Outcome, ...] = (
|
|
45
|
+
"fix_on_package_repo",
|
|
46
|
+
"fix_repo_unconfirmed",
|
|
47
|
+
"non_source_only",
|
|
48
|
+
"other_repo_only",
|
|
49
|
+
"unmerged_pr_only",
|
|
50
|
+
"unresolvable",
|
|
51
|
+
)
|
|
52
|
+
Verdict = Literal["met", "inconclusive", "not_met"]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class CandidateCheck(BaseModel):
|
|
56
|
+
model_config = ConfigDict(frozen=True)
|
|
57
|
+
|
|
58
|
+
kind: str
|
|
59
|
+
url: str
|
|
60
|
+
status: CandidateStatus
|
|
61
|
+
repo: str | None = None
|
|
62
|
+
on_package_repo: bool | None = None
|
|
63
|
+
files: int = 0
|
|
64
|
+
python_files: int = 0
|
|
65
|
+
native_files: int = 0
|
|
66
|
+
other_source_files: int = 0
|
|
67
|
+
truncated: bool = False
|
|
68
|
+
error: str | None = None
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def source_files(self) -> int:
|
|
72
|
+
return self.python_files + self.native_files + self.other_source_files
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class GroupCheck(BaseModel):
|
|
76
|
+
model_config = ConfigDict(frozen=True)
|
|
77
|
+
|
|
78
|
+
key: str
|
|
79
|
+
advisory_ids: tuple[str, ...]
|
|
80
|
+
packages: tuple[str, ...]
|
|
81
|
+
package_repos: tuple[str, ...]
|
|
82
|
+
outcome: Outcome
|
|
83
|
+
fix_touches_python: bool
|
|
84
|
+
candidates: tuple[CandidateCheck, ...]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class Estimate(BaseModel):
|
|
88
|
+
model_config = ConfigDict(frozen=True)
|
|
89
|
+
|
|
90
|
+
share: float
|
|
91
|
+
low: float
|
|
92
|
+
high: float
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class FixSampleReport(BaseModel):
|
|
96
|
+
as_of: date
|
|
97
|
+
seed: int
|
|
98
|
+
vulnerabilities: int
|
|
99
|
+
population: int
|
|
100
|
+
gate_share: float
|
|
101
|
+
sample_size: int
|
|
102
|
+
outcomes: dict[str, int]
|
|
103
|
+
real_fix_strict: Share
|
|
104
|
+
real_fix_lenient: Share
|
|
105
|
+
estimated_share_strict: Estimate
|
|
106
|
+
estimated_share_lenient: Estimate
|
|
107
|
+
verdict: Verdict
|
|
108
|
+
fixes_touching_python: Share
|
|
109
|
+
files_per_fix_median: float
|
|
110
|
+
files_per_fix_p90: float
|
|
111
|
+
truncated_fixes: int
|
|
112
|
+
candidates_checked: int
|
|
113
|
+
candidate_status: dict[str, int]
|
|
114
|
+
groups: tuple[GroupCheck, ...]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _count(change: ChangeSet) -> dict[str, int]:
|
|
118
|
+
kinds = Counter(classify_path(f.filename) for f in change.files)
|
|
119
|
+
return {
|
|
120
|
+
"files": len(change.files),
|
|
121
|
+
"python_files": kinds["python"],
|
|
122
|
+
"native_files": kinds["native"],
|
|
123
|
+
"other_source_files": kinds["other_source"],
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def check_candidate(
|
|
128
|
+
github: GitHubClient, candidate: FixCandidate, package_repos: frozenset[str]
|
|
129
|
+
) -> CandidateCheck:
|
|
130
|
+
"""Resolve one GitHub candidate. Rate-limit errors propagate and stop the run."""
|
|
131
|
+
base = {"kind": candidate.kind, "url": candidate.url}
|
|
132
|
+
repo = candidate.repo or ""
|
|
133
|
+
try:
|
|
134
|
+
if candidate.kind == "github_commit" and candidate.commit:
|
|
135
|
+
change = github.commit(repo, candidate.commit)
|
|
136
|
+
elif candidate.kind == "github_pull" and candidate.number is not None:
|
|
137
|
+
pull = github.pull(repo, candidate.number)
|
|
138
|
+
if not pull.merged:
|
|
139
|
+
return CandidateCheck(**base, status="unmerged", repo=pull.repo)
|
|
140
|
+
change = github.pull_files(repo, candidate.number)
|
|
141
|
+
elif candidate.kind == "github_compare" and candidate.base and candidate.head:
|
|
142
|
+
change = github.compare(repo, candidate.base, candidate.head)
|
|
143
|
+
else:
|
|
144
|
+
return CandidateCheck(**base, status="error", error="not a GitHub candidate")
|
|
145
|
+
except RateLimitError:
|
|
146
|
+
raise
|
|
147
|
+
except NotFoundError:
|
|
148
|
+
return CandidateCheck(**base, status="not_found")
|
|
149
|
+
except (HttpError, KeyError, ValueError) as error:
|
|
150
|
+
return CandidateCheck(**base, status="error", error=f"{type(error).__name__}: {error}")
|
|
151
|
+
on_repo = (change.repo in package_repos) if package_repos else None
|
|
152
|
+
return CandidateCheck(
|
|
153
|
+
**base,
|
|
154
|
+
status="resolved",
|
|
155
|
+
repo=change.repo,
|
|
156
|
+
on_package_repo=on_repo,
|
|
157
|
+
truncated=change.truncated,
|
|
158
|
+
**_count(change),
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def classify_group(checks: Sequence[CandidateCheck]) -> Outcome:
|
|
163
|
+
"""The best outcome among a vulnerability's candidates (see ``OUTCOME_ORDER``)."""
|
|
164
|
+
found: set[Outcome] = set()
|
|
165
|
+
for check in checks:
|
|
166
|
+
if check.status == "resolved":
|
|
167
|
+
if check.on_package_repo is False:
|
|
168
|
+
found.add("other_repo_only")
|
|
169
|
+
elif check.source_files == 0:
|
|
170
|
+
found.add("non_source_only")
|
|
171
|
+
elif check.on_package_repo:
|
|
172
|
+
found.add("fix_on_package_repo")
|
|
173
|
+
else:
|
|
174
|
+
found.add("fix_repo_unconfirmed")
|
|
175
|
+
elif check.status == "unmerged":
|
|
176
|
+
found.add("unmerged_pr_only")
|
|
177
|
+
else:
|
|
178
|
+
found.add("unresolvable")
|
|
179
|
+
return next(o for o in OUTCOME_ORDER if o in found)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def package_repos(
|
|
183
|
+
group: AliasGroup, by_id: dict[str, Advisory], github: GitHubClient, pypi: PyPIClient
|
|
184
|
+
) -> tuple[tuple[str, ...], frozenset[str]]:
|
|
185
|
+
"""The group's package names, and their canonical GitHub repositories."""
|
|
186
|
+
packages = sorted({p.name for i in group.advisory_ids for p in by_id[i].packages})
|
|
187
|
+
declared: set[str] = set()
|
|
188
|
+
for package in packages:
|
|
189
|
+
declared.update(pypi.declared_github_repos(package) or ())
|
|
190
|
+
for advisory_id in group.advisory_ids:
|
|
191
|
+
for candidate in by_id[advisory_id].fix_candidates:
|
|
192
|
+
if candidate.origin == "git_range" and candidate.kind == "github_commit":
|
|
193
|
+
declared.add(candidate.repo or "")
|
|
194
|
+
canonical: set[str] = set()
|
|
195
|
+
for repo in sorted(r for r in declared if r):
|
|
196
|
+
try:
|
|
197
|
+
canonical.add(github.canonical_repo(repo))
|
|
198
|
+
except NotFoundError:
|
|
199
|
+
continue
|
|
200
|
+
return tuple(packages), frozenset(canonical)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def check_group(
|
|
204
|
+
group: AliasGroup, by_id: dict[str, Advisory], github: GitHubClient, pypi: PyPIClient
|
|
205
|
+
) -> GroupCheck:
|
|
206
|
+
packages, repos = package_repos(group, by_id, github, pypi)
|
|
207
|
+
seen: set[tuple[str, str | None, str | None, int | None]] = set()
|
|
208
|
+
checks = []
|
|
209
|
+
for advisory_id in group.advisory_ids:
|
|
210
|
+
for candidate in by_id[advisory_id].fix_candidates:
|
|
211
|
+
identity = (candidate.kind, candidate.repo, candidate.commit, candidate.number)
|
|
212
|
+
if candidate.kind not in GITHUB_KINDS or identity in seen:
|
|
213
|
+
continue
|
|
214
|
+
seen.add(identity)
|
|
215
|
+
checks.append(check_candidate(github, candidate, repos))
|
|
216
|
+
outcome = classify_group(checks)
|
|
217
|
+
fixes = [c for c in checks if c.status == "resolved" and c.on_package_repo and c.source_files]
|
|
218
|
+
return GroupCheck(
|
|
219
|
+
key=group.key,
|
|
220
|
+
advisory_ids=group.advisory_ids,
|
|
221
|
+
packages=packages,
|
|
222
|
+
package_repos=tuple(sorted(repos)),
|
|
223
|
+
outcome=outcome,
|
|
224
|
+
fix_touches_python=any(c.python_files for c in fixes),
|
|
225
|
+
candidates=tuple(checks),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def draw_sample(groups: Iterable[AliasGroup], size: int, seed: int) -> list[AliasGroup]:
|
|
230
|
+
"""A seeded random sample, drawn from groups sorted by key so it is reproducible."""
|
|
231
|
+
ordered = sorted(groups, key=lambda g: g.key)
|
|
232
|
+
return random.Random(seed).sample(ordered, min(size, len(ordered))) # noqa: S311
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _scaled(gate_share: float, s: Share) -> Estimate:
|
|
236
|
+
return Estimate(
|
|
237
|
+
share=gate_share * s.share, low=gate_share * s.ci95_low, high=gate_share * s.ci95_high
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def build_report(
|
|
242
|
+
checks: Sequence[GroupCheck],
|
|
243
|
+
*,
|
|
244
|
+
as_of: date,
|
|
245
|
+
seed: int,
|
|
246
|
+
vulnerabilities: int,
|
|
247
|
+
population: int,
|
|
248
|
+
) -> FixSampleReport:
|
|
249
|
+
n = len(checks)
|
|
250
|
+
outcomes = Counter(c.outcome for c in checks)
|
|
251
|
+
strict = wilson(outcomes["fix_on_package_repo"], n)
|
|
252
|
+
lenient = wilson(outcomes["fix_on_package_repo"] + outcomes["fix_repo_unconfirmed"], n)
|
|
253
|
+
gate_share = population / vulnerabilities if vulnerabilities else 0.0
|
|
254
|
+
est_strict, est_lenient = _scaled(gate_share, strict), _scaled(gate_share, lenient)
|
|
255
|
+
if est_strict.share >= GATE_THRESHOLD:
|
|
256
|
+
verdict: Verdict = "met"
|
|
257
|
+
elif est_lenient.share >= GATE_THRESHOLD:
|
|
258
|
+
verdict = "inconclusive"
|
|
259
|
+
else:
|
|
260
|
+
verdict = "not_met"
|
|
261
|
+
|
|
262
|
+
fixed = [c for c in checks if c.outcome == "fix_on_package_repo"]
|
|
263
|
+
sizes = sorted(
|
|
264
|
+
max(x.files for x in c.candidates if x.status == "resolved" and x.on_package_repo)
|
|
265
|
+
for c in fixed
|
|
266
|
+
)
|
|
267
|
+
p90 = float(sizes[min(len(sizes) - 1, math.ceil(0.9 * len(sizes)) - 1)]) if sizes else 0.0
|
|
268
|
+
all_candidates = [x for c in checks for x in c.candidates]
|
|
269
|
+
return FixSampleReport(
|
|
270
|
+
as_of=as_of,
|
|
271
|
+
seed=seed,
|
|
272
|
+
vulnerabilities=vulnerabilities,
|
|
273
|
+
population=population,
|
|
274
|
+
gate_share=gate_share,
|
|
275
|
+
sample_size=n,
|
|
276
|
+
outcomes={o: outcomes[o] for o in OUTCOME_ORDER},
|
|
277
|
+
real_fix_strict=strict,
|
|
278
|
+
real_fix_lenient=lenient,
|
|
279
|
+
estimated_share_strict=est_strict,
|
|
280
|
+
estimated_share_lenient=est_lenient,
|
|
281
|
+
verdict=verdict,
|
|
282
|
+
fixes_touching_python=wilson(sum(c.fix_touches_python for c in fixed), len(fixed)),
|
|
283
|
+
files_per_fix_median=float(statistics.median(sizes)) if sizes else 0.0,
|
|
284
|
+
files_per_fix_p90=p90,
|
|
285
|
+
truncated_fixes=sum(
|
|
286
|
+
any(x.truncated for x in c.candidates if x.on_package_repo) for c in fixed
|
|
287
|
+
),
|
|
288
|
+
candidates_checked=len(all_candidates),
|
|
289
|
+
candidate_status=dict(sorted(Counter(x.status for x in all_candidates).items())),
|
|
290
|
+
groups=tuple(checks),
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
_OUTCOME_TEXT: dict[str, str] = {
|
|
295
|
+
"fix_on_package_repo": "Resolves on the package's own repo and changes non-test source",
|
|
296
|
+
"fix_repo_unconfirmed": "Changes non-test source; package repo not declared anywhere",
|
|
297
|
+
"non_source_only": "Resolves, but changes only tests, docs or metadata",
|
|
298
|
+
"other_repo_only": "Resolves, but only on repos that are not the package's",
|
|
299
|
+
"unmerged_pr_only": "Only an unmerged pull request",
|
|
300
|
+
"unresolvable": "No candidate resolves (deleted, private or malformed)",
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _pct(s: Share) -> str:
|
|
305
|
+
return f"{s.share:.1%} ({s.count} / {s.total}; 95% CI {s.ci95_low:.1%} to {s.ci95_high:.1%})"
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _est(e: Estimate) -> str:
|
|
309
|
+
return f"{e.share:.1%} (95% CI {e.low:.1%} to {e.high:.1%})"
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def render_markdown(report: FixSampleReport, manifest_lines: Sequence[str]) -> str:
|
|
313
|
+
verdict_text = {
|
|
314
|
+
"met": "**Condition met.** The real fix share is above the 40% threshold, so the "
|
|
315
|
+
"fix-commit locator agent stays optional.",
|
|
316
|
+
"inconclusive": "**Inconclusive.** Only the lenient estimate clears 40%; the owner "
|
|
317
|
+
"decides whether to promote the fix-commit locator agent.",
|
|
318
|
+
"not_met": "**Condition not met.** The real fix share is below 40%, so the fix-commit "
|
|
319
|
+
"locator agent becomes essential (proposal §15).",
|
|
320
|
+
}[report.verdict]
|
|
321
|
+
rows = [f"| {_OUTCOME_TEXT[o]} | {report.outcomes[o]} |" for o in OUTCOME_ORDER]
|
|
322
|
+
return "\n".join(
|
|
323
|
+
[
|
|
324
|
+
"# Fix-candidate sample check",
|
|
325
|
+
"",
|
|
326
|
+
f"The condition attached to the Week 1 gate. Snapshot of {report.as_of.isoformat()}; "
|
|
327
|
+
"generated by `python -m patchsight.pipeline fix-sample`; do not edit by hand.",
|
|
328
|
+
"",
|
|
329
|
+
"## Sources",
|
|
330
|
+
"",
|
|
331
|
+
*manifest_lines,
|
|
332
|
+
"",
|
|
333
|
+
"Advisory data: OSV.dev, the GitHub Advisory Database and the PyPA Advisory "
|
|
334
|
+
"Database (all CC BY 4.0). Repository data: the GitHub REST API and PyPI "
|
|
335
|
+
"metadata. This report holds counts only; no third-party code.",
|
|
336
|
+
"",
|
|
337
|
+
"## Result",
|
|
338
|
+
"",
|
|
339
|
+
verdict_text,
|
|
340
|
+
"",
|
|
341
|
+
f"- Population: {report.population:,} of {report.vulnerabilities:,} vulnerabilities "
|
|
342
|
+
f"have a GitHub fix candidate ({report.gate_share:.1%}, the Week 1 gate metric).",
|
|
343
|
+
f"- Random sample: {report.sample_size} of them, seed {report.seed}.",
|
|
344
|
+
f"- **Strict:** a real fix on the package's own repo: {_pct(report.real_fix_strict)}.",
|
|
345
|
+
f"- **Lenient:** also counting fixes whose package repo is not declared: "
|
|
346
|
+
f"{_pct(report.real_fix_lenient)}.",
|
|
347
|
+
"",
|
|
348
|
+
"Estimated share of *all* PyPI vulnerabilities with a real fix commit "
|
|
349
|
+
"(gate metric times sample share; the gate metric's own ±1.1-point interval is "
|
|
350
|
+
"ignored):",
|
|
351
|
+
"",
|
|
352
|
+
f"- Strict: **{_est(report.estimated_share_strict)}**",
|
|
353
|
+
f"- Lenient: {_est(report.estimated_share_lenient)}",
|
|
354
|
+
"",
|
|
355
|
+
"## Outcomes",
|
|
356
|
+
"",
|
|
357
|
+
"Each vulnerability gets the best outcome among its candidates.",
|
|
358
|
+
"",
|
|
359
|
+
"| Outcome | Vulnerabilities |",
|
|
360
|
+
"|---|---|",
|
|
361
|
+
*rows,
|
|
362
|
+
"",
|
|
363
|
+
f"Candidates checked: {report.candidates_checked} "
|
|
364
|
+
f"({', '.join(f'{k} {v}' for k, v in report.candidate_status.items())}).",
|
|
365
|
+
"",
|
|
366
|
+
"## What the real fixes look like",
|
|
367
|
+
"",
|
|
368
|
+
f"- Fixes that change Python source (what tree-sitter maps to functions): "
|
|
369
|
+
f"{_pct(report.fixes_touching_python)}. The rest change only native code "
|
|
370
|
+
"(C, Rust and so on) or templates and scripts, so they fall back to *unknown*.",
|
|
371
|
+
f"- Files changed per fix, including tests and docs: median "
|
|
372
|
+
f"{report.files_per_fix_median:.0f}, p90 {report.files_per_fix_p90:.0f}.",
|
|
373
|
+
f"- Fixes whose file list GitHub truncated: {report.truncated_fixes}.",
|
|
374
|
+
"",
|
|
375
|
+
"## Limits of this check",
|
|
376
|
+
"",
|
|
377
|
+
"- These are necessary conditions, not proof. A commit on the right repo that "
|
|
378
|
+
"changes source may still be a refactor rather than the fix; that is measured "
|
|
379
|
+
"against labelled data later.",
|
|
380
|
+
"- A package whose metadata names no GitHub repo and has no OSV GIT range lands "
|
|
381
|
+
"in *repo not declared*, even when the candidate is correct.",
|
|
382
|
+
"",
|
|
383
|
+
]
|
|
384
|
+
)
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""A JSON-over-HTTP client with an on-disk response cache, retries and typed errors.
|
|
2
|
+
|
|
3
|
+
Offline pipeline only. Responses, including "not found", are cached by request, so
|
|
4
|
+
re-runs make no network calls and give the same answers. The cache can hold third-party
|
|
5
|
+
code (diff patches), so it lives in an ignored directory and is never committed.
|
|
6
|
+
Credentials are sent as headers but never become part of a cache key.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Callable, Mapping
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
from tenacity import (
|
|
18
|
+
Retrying,
|
|
19
|
+
retry_if_exception_type,
|
|
20
|
+
stop_after_attempt,
|
|
21
|
+
wait_exponential,
|
|
22
|
+
)
|
|
23
|
+
from tenacity.wait import wait_base
|
|
24
|
+
|
|
25
|
+
NOT_FOUND_STATUSES = frozenset({404, 409, 410, 422, 451})
|
|
26
|
+
RETRY_STATUSES = frozenset({500, 502, 503, 504})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class HttpError(RuntimeError):
|
|
30
|
+
"""A request failed in a way that retrying will not fix."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class NotFoundError(HttpError):
|
|
34
|
+
"""The resource does not exist, or never will for this request."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RateLimitError(HttpError):
|
|
38
|
+
"""The server's rate limit is exhausted."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, message: str, reset_at: float | None = None) -> None:
|
|
41
|
+
super().__init__(message)
|
|
42
|
+
self.reset_at = reset_at
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class _RetryableError(Exception):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class CachedJsonClient:
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
base_url: str,
|
|
53
|
+
cache_dir: Path,
|
|
54
|
+
headers: Mapping[str, str] | None = None,
|
|
55
|
+
transport: httpx.BaseTransport | None = None,
|
|
56
|
+
max_attempts: int = 4,
|
|
57
|
+
wait: wait_base | None = None,
|
|
58
|
+
wait_for_rate_limit: float = 0.0,
|
|
59
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
60
|
+
clock: Callable[[], float] = time.time,
|
|
61
|
+
) -> None:
|
|
62
|
+
"""``wait_for_rate_limit``: the longest wait, in seconds, for an exhausted rate
|
|
63
|
+
limit to reset before retrying. Zero raises ``RateLimitError`` at once."""
|
|
64
|
+
self._cache_dir = cache_dir
|
|
65
|
+
self._max_attempts = max_attempts
|
|
66
|
+
self._wait_for_rate_limit = wait_for_rate_limit
|
|
67
|
+
self._sleep = sleep
|
|
68
|
+
self._clock = clock
|
|
69
|
+
self.rate_limit_waits = 0
|
|
70
|
+
self._http = httpx.Client(
|
|
71
|
+
base_url=base_url,
|
|
72
|
+
headers=dict(headers or {}),
|
|
73
|
+
timeout=httpx.Timeout(30.0, connect=10.0),
|
|
74
|
+
follow_redirects=True,
|
|
75
|
+
transport=transport,
|
|
76
|
+
)
|
|
77
|
+
self._retrying: Callable[[], Retrying] = lambda: Retrying(
|
|
78
|
+
retry=retry_if_exception_type(_RetryableError),
|
|
79
|
+
stop=stop_after_attempt(max_attempts),
|
|
80
|
+
wait=wait or wait_exponential(multiplier=1, max=30),
|
|
81
|
+
reraise=True,
|
|
82
|
+
)
|
|
83
|
+
self.network_requests = 0
|
|
84
|
+
|
|
85
|
+
def close(self) -> None:
|
|
86
|
+
self._http.close()
|
|
87
|
+
|
|
88
|
+
def _cache_path(self, path: str, params: Mapping[str, str | int]) -> Path:
|
|
89
|
+
query = "&".join(f"{k}={params[k]}" for k in sorted(params))
|
|
90
|
+
key = hashlib.sha256(f"{self._http.base_url}{path}?{query}".encode()).hexdigest()
|
|
91
|
+
return self._cache_dir / key[:2] / f"{key}.json"
|
|
92
|
+
|
|
93
|
+
def get_json(self, path: str, params: Mapping[str, str | int] | None = None) -> Any:
|
|
94
|
+
params = dict(params or {})
|
|
95
|
+
cache_file = self._cache_path(path, params)
|
|
96
|
+
if cache_file.exists():
|
|
97
|
+
cached = json.loads(cache_file.read_text(encoding="utf-8"))
|
|
98
|
+
else:
|
|
99
|
+
cached = self._fetch(path, params)
|
|
100
|
+
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
cache_file.write_text(json.dumps(cached), encoding="utf-8")
|
|
102
|
+
if cached["status"] != 200:
|
|
103
|
+
raise NotFoundError(f"{path}: HTTP {cached['status']}")
|
|
104
|
+
return cached["body"]
|
|
105
|
+
|
|
106
|
+
def _fetch(self, path: str, params: Mapping[str, str | int]) -> dict[str, Any]:
|
|
107
|
+
while True:
|
|
108
|
+
try:
|
|
109
|
+
for attempt in self._retrying():
|
|
110
|
+
with attempt:
|
|
111
|
+
return self._fetch_once(path, params)
|
|
112
|
+
except _RetryableError as error:
|
|
113
|
+
# Out of retries: an ordinary, catchable failure, not an internal type.
|
|
114
|
+
raise HttpError(f"gave up after {self._max_attempts} attempts: {error}") from None
|
|
115
|
+
except RateLimitError as limited:
|
|
116
|
+
if limited.reset_at is None:
|
|
117
|
+
raise
|
|
118
|
+
wait = limited.reset_at - self._clock() + 5.0
|
|
119
|
+
if not 0 < wait <= self._wait_for_rate_limit:
|
|
120
|
+
raise
|
|
121
|
+
self.rate_limit_waits += 1
|
|
122
|
+
self._sleep(wait)
|
|
123
|
+
else: # pragma: no cover - the loop above always returns or raises
|
|
124
|
+
raise AssertionError("unreachable")
|
|
125
|
+
|
|
126
|
+
def _fetch_once(self, path: str, params: Mapping[str, str | int]) -> dict[str, Any]:
|
|
127
|
+
self.network_requests += 1
|
|
128
|
+
try:
|
|
129
|
+
response = self._http.get(path, params=dict(params))
|
|
130
|
+
except httpx.TransportError as error:
|
|
131
|
+
raise _RetryableError(str(error)) from error
|
|
132
|
+
status = response.status_code
|
|
133
|
+
if status == 200:
|
|
134
|
+
return {"status": 200, "body": response.json()}
|
|
135
|
+
if status in NOT_FOUND_STATUSES:
|
|
136
|
+
return {"status": status, "body": None}
|
|
137
|
+
if status in (403, 429) and response.headers.get("x-ratelimit-remaining") == "0":
|
|
138
|
+
reset = response.headers.get("x-ratelimit-reset")
|
|
139
|
+
reset_at = float(reset) if reset and reset.isdigit() else None
|
|
140
|
+
raise RateLimitError(f"{path}: rate limit exhausted until {reset}", reset_at)
|
|
141
|
+
if status in RETRY_STATUSES or status == 429:
|
|
142
|
+
raise _RetryableError(f"{path}: HTTP {status}")
|
|
143
|
+
raise HttpError(f"{path}: HTTP {status}")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Ingestion: source snapshots, OSV normalisation, fix-commit references, alias groups."""
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Group advisories that describe the same vulnerability.
|
|
2
|
+
|
|
3
|
+
The same vulnerability usually appears several times in the PyPI dump, for example as a
|
|
4
|
+
GHSA record and a PYSEC record that alias each other or share a CVE. Counting and
|
|
5
|
+
extraction work per group, so it is done once per vulnerability, not once per record.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Iterable
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel, ConfigDict
|
|
12
|
+
|
|
13
|
+
from patchsight.pipeline.ingest.osv import Advisory
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AliasGroup(BaseModel):
|
|
17
|
+
model_config = ConfigDict(frozen=True)
|
|
18
|
+
|
|
19
|
+
key: str
|
|
20
|
+
advisory_ids: tuple[str, ...]
|
|
21
|
+
identifiers: tuple[str, ...]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _UnionFind:
|
|
25
|
+
def __init__(self) -> None:
|
|
26
|
+
self.parent: dict[str, str] = {}
|
|
27
|
+
|
|
28
|
+
def find(self, item: str) -> str:
|
|
29
|
+
self.parent.setdefault(item, item)
|
|
30
|
+
root = item
|
|
31
|
+
while self.parent[root] != root:
|
|
32
|
+
root = self.parent[root]
|
|
33
|
+
while self.parent[item] != root:
|
|
34
|
+
self.parent[item], item = root, self.parent[item]
|
|
35
|
+
return root
|
|
36
|
+
|
|
37
|
+
def union(self, a: str, b: str) -> None:
|
|
38
|
+
root_a, root_b = self.find(a), self.find(b)
|
|
39
|
+
if root_a != root_b:
|
|
40
|
+
low, high = sorted((root_a, root_b))
|
|
41
|
+
self.parent[high] = low
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def alias_groups(advisories: Iterable[Advisory]) -> list[AliasGroup]:
|
|
45
|
+
"""Group advisories connected by any shared identifier. Independent of input order.
|
|
46
|
+
|
|
47
|
+
The group key is the smallest advisory ID in the group, so it is deterministic.
|
|
48
|
+
"""
|
|
49
|
+
union = _UnionFind()
|
|
50
|
+
advisory_ids: set[str] = set()
|
|
51
|
+
for advisory in advisories:
|
|
52
|
+
advisory_ids.add(advisory.id)
|
|
53
|
+
union.find(advisory.id)
|
|
54
|
+
for alias in advisory.aliases:
|
|
55
|
+
union.union(advisory.id, alias)
|
|
56
|
+
|
|
57
|
+
members: dict[str, list[str]] = {}
|
|
58
|
+
for identifier in union.parent:
|
|
59
|
+
members.setdefault(union.find(identifier), []).append(identifier)
|
|
60
|
+
|
|
61
|
+
groups = []
|
|
62
|
+
for identifiers in members.values():
|
|
63
|
+
ids = sorted(i for i in identifiers if i in advisory_ids)
|
|
64
|
+
if ids:
|
|
65
|
+
groups.append(
|
|
66
|
+
AliasGroup(
|
|
67
|
+
key=ids[0], advisory_ids=tuple(ids), identifiers=tuple(sorted(identifiers))
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
return sorted(groups, key=lambda group: group.key)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def first_published(group: AliasGroup, by_id: dict[str, Advisory]) -> datetime | None:
|
|
74
|
+
"""The earliest ``published`` timestamp among the group's advisories."""
|
|
75
|
+
dates = [d for i in group.advisory_ids if (d := by_id[i].published) is not None]
|
|
76
|
+
return min(dates) if dates else None
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Partial, sparse clone of ``github/advisory-database``, limited to PyPI advisories.
|
|
2
|
+
|
|
3
|
+
The full repository is several gigabytes. GHSA files live at
|
|
4
|
+
``advisories/github-reviewed/<YYYY>/<MM>/<ID>/<ID>.json``, keyed by publication month, so
|
|
5
|
+
the PyPI subset can be checked out directly from the OSV records. Cone-mode sparse
|
|
6
|
+
checkout keeps this to seconds; non-cone patterns are matched against every path and are
|
|
7
|
+
orders of magnitude slower.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import subprocess
|
|
12
|
+
from collections.abc import Iterable
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, ConfigDict
|
|
16
|
+
|
|
17
|
+
from patchsight.pipeline.ingest.osv import Advisory
|
|
18
|
+
|
|
19
|
+
GHSA_REPO_URL = "https://github.com/github/advisory-database.git"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class GhsaScan(BaseModel):
|
|
23
|
+
model_config = ConfigDict(frozen=True)
|
|
24
|
+
|
|
25
|
+
expected: int
|
|
26
|
+
found: int
|
|
27
|
+
missing: tuple[str, ...]
|
|
28
|
+
with_symbol_data: tuple[str, ...]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def ghsa_directory(advisory: Advisory) -> str | None:
|
|
32
|
+
if advisory.source != "GHSA" or advisory.published is None:
|
|
33
|
+
return None
|
|
34
|
+
month = advisory.published.strftime("%Y/%m")
|
|
35
|
+
return f"advisories/github-reviewed/{month}/{advisory.id}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _git(*args: str) -> None:
|
|
39
|
+
subprocess.run(["git", *args], check=True) # noqa: S603, S607 - fixed argv, no shell
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def sparse_clone(dest: Path, advisories: Iterable[Advisory]) -> None:
|
|
43
|
+
"""Clone only the directories of the given GHSA advisories into ``dest``."""
|
|
44
|
+
directories = sorted({d for a in advisories if (d := ghsa_directory(a))})
|
|
45
|
+
if not (dest / ".git").exists():
|
|
46
|
+
_git("clone", "-q", "--depth", "1", "--filter=blob:none", "--no-checkout",
|
|
47
|
+
GHSA_REPO_URL, str(dest)) # fmt: skip
|
|
48
|
+
subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
49
|
+
["git", "-C", str(dest), "sparse-checkout", "set", "--cone", "--stdin"], # noqa: S607
|
|
50
|
+
input="\n".join(directories),
|
|
51
|
+
text=True,
|
|
52
|
+
check=True,
|
|
53
|
+
)
|
|
54
|
+
_git("-C", str(dest), "checkout", "-q", "main")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def scan(repo: Path, advisories: Iterable[Advisory]) -> GhsaScan:
|
|
58
|
+
"""Check which GHSA advisories in the clone carry ``ecosystem_specific`` symbol data."""
|
|
59
|
+
expected, missing, with_symbols = 0, [], []
|
|
60
|
+
for advisory in advisories:
|
|
61
|
+
directory = ghsa_directory(advisory)
|
|
62
|
+
if directory is None:
|
|
63
|
+
continue
|
|
64
|
+
expected += 1
|
|
65
|
+
path = repo / directory / f"{advisory.id}.json"
|
|
66
|
+
if not path.exists():
|
|
67
|
+
missing.append(advisory.id)
|
|
68
|
+
continue
|
|
69
|
+
record = json.loads(path.read_text(encoding="utf-8"))
|
|
70
|
+
for affected in record.get("affected", []):
|
|
71
|
+
specific = affected.get("ecosystem_specific") or {}
|
|
72
|
+
if specific.get("affected_functions") or specific.get("imports"):
|
|
73
|
+
with_symbols.append(advisory.id)
|
|
74
|
+
break
|
|
75
|
+
return GhsaScan(
|
|
76
|
+
expected=expected,
|
|
77
|
+
found=expected - len(missing),
|
|
78
|
+
missing=tuple(sorted(missing)),
|
|
79
|
+
with_symbol_data=tuple(sorted(with_symbols)),
|
|
80
|
+
)
|