patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,384 @@
1
+ """Check a random sample of fix candidates against GitHub: are they really fixes?
2
+
3
+ This is the condition attached to passing the Week 1 gate. A link to a commit is only a
4
+ *candidate*. A vulnerability counts as having a real fix only when a candidate:
5
+
6
+ 1. resolves on GitHub (a commit, a merged pull request or a tag comparison);
7
+ 2. sits on the package's own repository, as declared in its PyPI metadata or an OSV
8
+ ``GIT`` range (renames followed); and
9
+ 3. changes at least one non-test source file.
10
+
11
+ These are necessary conditions checked by code, with no LLM and no hand labelling. They
12
+ do not prove a commit is *the* fix; whether the right functions changed is measured later
13
+ against labelled data.
14
+ """
15
+
16
+ import math
17
+ import random
18
+ import statistics
19
+ from collections import Counter
20
+ from collections.abc import Iterable, Sequence
21
+ from datetime import date
22
+ from typing import Literal
23
+
24
+ from pydantic import BaseModel, ConfigDict
25
+
26
+ from patchsight.pipeline.diffs.github import ChangeSet, GitHubClient
27
+ from patchsight.pipeline.diffs.paths import classify_path
28
+ from patchsight.pipeline.http import HttpError, NotFoundError, RateLimitError
29
+ from patchsight.pipeline.ingest.aliases import AliasGroup
30
+ from patchsight.pipeline.ingest.osv import Advisory
31
+ from patchsight.pipeline.ingest.references import FixCandidate
32
+ from patchsight.pipeline.pypi import PyPIClient
33
+ from patchsight.pipeline.week1 import GATE_THRESHOLD, GITHUB_KINDS, Share, wilson
34
+
35
+ CandidateStatus = Literal["resolved", "not_found", "unmerged", "error"]
36
+ Outcome = Literal[
37
+ "fix_on_package_repo",
38
+ "fix_repo_unconfirmed",
39
+ "non_source_only",
40
+ "other_repo_only",
41
+ "unmerged_pr_only",
42
+ "unresolvable",
43
+ ]
44
+ OUTCOME_ORDER: tuple[Outcome, ...] = (
45
+ "fix_on_package_repo",
46
+ "fix_repo_unconfirmed",
47
+ "non_source_only",
48
+ "other_repo_only",
49
+ "unmerged_pr_only",
50
+ "unresolvable",
51
+ )
52
+ Verdict = Literal["met", "inconclusive", "not_met"]
53
+
54
+
55
+ class CandidateCheck(BaseModel):
56
+ model_config = ConfigDict(frozen=True)
57
+
58
+ kind: str
59
+ url: str
60
+ status: CandidateStatus
61
+ repo: str | None = None
62
+ on_package_repo: bool | None = None
63
+ files: int = 0
64
+ python_files: int = 0
65
+ native_files: int = 0
66
+ other_source_files: int = 0
67
+ truncated: bool = False
68
+ error: str | None = None
69
+
70
+ @property
71
+ def source_files(self) -> int:
72
+ return self.python_files + self.native_files + self.other_source_files
73
+
74
+
75
+ class GroupCheck(BaseModel):
76
+ model_config = ConfigDict(frozen=True)
77
+
78
+ key: str
79
+ advisory_ids: tuple[str, ...]
80
+ packages: tuple[str, ...]
81
+ package_repos: tuple[str, ...]
82
+ outcome: Outcome
83
+ fix_touches_python: bool
84
+ candidates: tuple[CandidateCheck, ...]
85
+
86
+
87
+ class Estimate(BaseModel):
88
+ model_config = ConfigDict(frozen=True)
89
+
90
+ share: float
91
+ low: float
92
+ high: float
93
+
94
+
95
+ class FixSampleReport(BaseModel):
96
+ as_of: date
97
+ seed: int
98
+ vulnerabilities: int
99
+ population: int
100
+ gate_share: float
101
+ sample_size: int
102
+ outcomes: dict[str, int]
103
+ real_fix_strict: Share
104
+ real_fix_lenient: Share
105
+ estimated_share_strict: Estimate
106
+ estimated_share_lenient: Estimate
107
+ verdict: Verdict
108
+ fixes_touching_python: Share
109
+ files_per_fix_median: float
110
+ files_per_fix_p90: float
111
+ truncated_fixes: int
112
+ candidates_checked: int
113
+ candidate_status: dict[str, int]
114
+ groups: tuple[GroupCheck, ...]
115
+
116
+
117
+ def _count(change: ChangeSet) -> dict[str, int]:
118
+ kinds = Counter(classify_path(f.filename) for f in change.files)
119
+ return {
120
+ "files": len(change.files),
121
+ "python_files": kinds["python"],
122
+ "native_files": kinds["native"],
123
+ "other_source_files": kinds["other_source"],
124
+ }
125
+
126
+
127
+ def check_candidate(
128
+ github: GitHubClient, candidate: FixCandidate, package_repos: frozenset[str]
129
+ ) -> CandidateCheck:
130
+ """Resolve one GitHub candidate. Rate-limit errors propagate and stop the run."""
131
+ base = {"kind": candidate.kind, "url": candidate.url}
132
+ repo = candidate.repo or ""
133
+ try:
134
+ if candidate.kind == "github_commit" and candidate.commit:
135
+ change = github.commit(repo, candidate.commit)
136
+ elif candidate.kind == "github_pull" and candidate.number is not None:
137
+ pull = github.pull(repo, candidate.number)
138
+ if not pull.merged:
139
+ return CandidateCheck(**base, status="unmerged", repo=pull.repo)
140
+ change = github.pull_files(repo, candidate.number)
141
+ elif candidate.kind == "github_compare" and candidate.base and candidate.head:
142
+ change = github.compare(repo, candidate.base, candidate.head)
143
+ else:
144
+ return CandidateCheck(**base, status="error", error="not a GitHub candidate")
145
+ except RateLimitError:
146
+ raise
147
+ except NotFoundError:
148
+ return CandidateCheck(**base, status="not_found")
149
+ except (HttpError, KeyError, ValueError) as error:
150
+ return CandidateCheck(**base, status="error", error=f"{type(error).__name__}: {error}")
151
+ on_repo = (change.repo in package_repos) if package_repos else None
152
+ return CandidateCheck(
153
+ **base,
154
+ status="resolved",
155
+ repo=change.repo,
156
+ on_package_repo=on_repo,
157
+ truncated=change.truncated,
158
+ **_count(change),
159
+ )
160
+
161
+
162
+ def classify_group(checks: Sequence[CandidateCheck]) -> Outcome:
163
+ """The best outcome among a vulnerability's candidates (see ``OUTCOME_ORDER``)."""
164
+ found: set[Outcome] = set()
165
+ for check in checks:
166
+ if check.status == "resolved":
167
+ if check.on_package_repo is False:
168
+ found.add("other_repo_only")
169
+ elif check.source_files == 0:
170
+ found.add("non_source_only")
171
+ elif check.on_package_repo:
172
+ found.add("fix_on_package_repo")
173
+ else:
174
+ found.add("fix_repo_unconfirmed")
175
+ elif check.status == "unmerged":
176
+ found.add("unmerged_pr_only")
177
+ else:
178
+ found.add("unresolvable")
179
+ return next(o for o in OUTCOME_ORDER if o in found)
180
+
181
+
182
+ def package_repos(
183
+ group: AliasGroup, by_id: dict[str, Advisory], github: GitHubClient, pypi: PyPIClient
184
+ ) -> tuple[tuple[str, ...], frozenset[str]]:
185
+ """The group's package names, and their canonical GitHub repositories."""
186
+ packages = sorted({p.name for i in group.advisory_ids for p in by_id[i].packages})
187
+ declared: set[str] = set()
188
+ for package in packages:
189
+ declared.update(pypi.declared_github_repos(package) or ())
190
+ for advisory_id in group.advisory_ids:
191
+ for candidate in by_id[advisory_id].fix_candidates:
192
+ if candidate.origin == "git_range" and candidate.kind == "github_commit":
193
+ declared.add(candidate.repo or "")
194
+ canonical: set[str] = set()
195
+ for repo in sorted(r for r in declared if r):
196
+ try:
197
+ canonical.add(github.canonical_repo(repo))
198
+ except NotFoundError:
199
+ continue
200
+ return tuple(packages), frozenset(canonical)
201
+
202
+
203
+ def check_group(
204
+ group: AliasGroup, by_id: dict[str, Advisory], github: GitHubClient, pypi: PyPIClient
205
+ ) -> GroupCheck:
206
+ packages, repos = package_repos(group, by_id, github, pypi)
207
+ seen: set[tuple[str, str | None, str | None, int | None]] = set()
208
+ checks = []
209
+ for advisory_id in group.advisory_ids:
210
+ for candidate in by_id[advisory_id].fix_candidates:
211
+ identity = (candidate.kind, candidate.repo, candidate.commit, candidate.number)
212
+ if candidate.kind not in GITHUB_KINDS or identity in seen:
213
+ continue
214
+ seen.add(identity)
215
+ checks.append(check_candidate(github, candidate, repos))
216
+ outcome = classify_group(checks)
217
+ fixes = [c for c in checks if c.status == "resolved" and c.on_package_repo and c.source_files]
218
+ return GroupCheck(
219
+ key=group.key,
220
+ advisory_ids=group.advisory_ids,
221
+ packages=packages,
222
+ package_repos=tuple(sorted(repos)),
223
+ outcome=outcome,
224
+ fix_touches_python=any(c.python_files for c in fixes),
225
+ candidates=tuple(checks),
226
+ )
227
+
228
+
229
+ def draw_sample(groups: Iterable[AliasGroup], size: int, seed: int) -> list[AliasGroup]:
230
+ """A seeded random sample, drawn from groups sorted by key so it is reproducible."""
231
+ ordered = sorted(groups, key=lambda g: g.key)
232
+ return random.Random(seed).sample(ordered, min(size, len(ordered))) # noqa: S311
233
+
234
+
235
+ def _scaled(gate_share: float, s: Share) -> Estimate:
236
+ return Estimate(
237
+ share=gate_share * s.share, low=gate_share * s.ci95_low, high=gate_share * s.ci95_high
238
+ )
239
+
240
+
241
+ def build_report(
242
+ checks: Sequence[GroupCheck],
243
+ *,
244
+ as_of: date,
245
+ seed: int,
246
+ vulnerabilities: int,
247
+ population: int,
248
+ ) -> FixSampleReport:
249
+ n = len(checks)
250
+ outcomes = Counter(c.outcome for c in checks)
251
+ strict = wilson(outcomes["fix_on_package_repo"], n)
252
+ lenient = wilson(outcomes["fix_on_package_repo"] + outcomes["fix_repo_unconfirmed"], n)
253
+ gate_share = population / vulnerabilities if vulnerabilities else 0.0
254
+ est_strict, est_lenient = _scaled(gate_share, strict), _scaled(gate_share, lenient)
255
+ if est_strict.share >= GATE_THRESHOLD:
256
+ verdict: Verdict = "met"
257
+ elif est_lenient.share >= GATE_THRESHOLD:
258
+ verdict = "inconclusive"
259
+ else:
260
+ verdict = "not_met"
261
+
262
+ fixed = [c for c in checks if c.outcome == "fix_on_package_repo"]
263
+ sizes = sorted(
264
+ max(x.files for x in c.candidates if x.status == "resolved" and x.on_package_repo)
265
+ for c in fixed
266
+ )
267
+ p90 = float(sizes[min(len(sizes) - 1, math.ceil(0.9 * len(sizes)) - 1)]) if sizes else 0.0
268
+ all_candidates = [x for c in checks for x in c.candidates]
269
+ return FixSampleReport(
270
+ as_of=as_of,
271
+ seed=seed,
272
+ vulnerabilities=vulnerabilities,
273
+ population=population,
274
+ gate_share=gate_share,
275
+ sample_size=n,
276
+ outcomes={o: outcomes[o] for o in OUTCOME_ORDER},
277
+ real_fix_strict=strict,
278
+ real_fix_lenient=lenient,
279
+ estimated_share_strict=est_strict,
280
+ estimated_share_lenient=est_lenient,
281
+ verdict=verdict,
282
+ fixes_touching_python=wilson(sum(c.fix_touches_python for c in fixed), len(fixed)),
283
+ files_per_fix_median=float(statistics.median(sizes)) if sizes else 0.0,
284
+ files_per_fix_p90=p90,
285
+ truncated_fixes=sum(
286
+ any(x.truncated for x in c.candidates if x.on_package_repo) for c in fixed
287
+ ),
288
+ candidates_checked=len(all_candidates),
289
+ candidate_status=dict(sorted(Counter(x.status for x in all_candidates).items())),
290
+ groups=tuple(checks),
291
+ )
292
+
293
+
294
+ _OUTCOME_TEXT: dict[str, str] = {
295
+ "fix_on_package_repo": "Resolves on the package's own repo and changes non-test source",
296
+ "fix_repo_unconfirmed": "Changes non-test source; package repo not declared anywhere",
297
+ "non_source_only": "Resolves, but changes only tests, docs or metadata",
298
+ "other_repo_only": "Resolves, but only on repos that are not the package's",
299
+ "unmerged_pr_only": "Only an unmerged pull request",
300
+ "unresolvable": "No candidate resolves (deleted, private or malformed)",
301
+ }
302
+
303
+
304
+ def _pct(s: Share) -> str:
305
+ return f"{s.share:.1%} ({s.count} / {s.total}; 95% CI {s.ci95_low:.1%} to {s.ci95_high:.1%})"
306
+
307
+
308
+ def _est(e: Estimate) -> str:
309
+ return f"{e.share:.1%} (95% CI {e.low:.1%} to {e.high:.1%})"
310
+
311
+
312
+ def render_markdown(report: FixSampleReport, manifest_lines: Sequence[str]) -> str:
313
+ verdict_text = {
314
+ "met": "**Condition met.** The real fix share is above the 40% threshold, so the "
315
+ "fix-commit locator agent stays optional.",
316
+ "inconclusive": "**Inconclusive.** Only the lenient estimate clears 40%; the owner "
317
+ "decides whether to promote the fix-commit locator agent.",
318
+ "not_met": "**Condition not met.** The real fix share is below 40%, so the fix-commit "
319
+ "locator agent becomes essential (proposal §15).",
320
+ }[report.verdict]
321
+ rows = [f"| {_OUTCOME_TEXT[o]} | {report.outcomes[o]} |" for o in OUTCOME_ORDER]
322
+ return "\n".join(
323
+ [
324
+ "# Fix-candidate sample check",
325
+ "",
326
+ f"The condition attached to the Week 1 gate. Snapshot of {report.as_of.isoformat()}; "
327
+ "generated by `python -m patchsight.pipeline fix-sample`; do not edit by hand.",
328
+ "",
329
+ "## Sources",
330
+ "",
331
+ *manifest_lines,
332
+ "",
333
+ "Advisory data: OSV.dev, the GitHub Advisory Database and the PyPA Advisory "
334
+ "Database (all CC BY 4.0). Repository data: the GitHub REST API and PyPI "
335
+ "metadata. This report holds counts only; no third-party code.",
336
+ "",
337
+ "## Result",
338
+ "",
339
+ verdict_text,
340
+ "",
341
+ f"- Population: {report.population:,} of {report.vulnerabilities:,} vulnerabilities "
342
+ f"have a GitHub fix candidate ({report.gate_share:.1%}, the Week 1 gate metric).",
343
+ f"- Random sample: {report.sample_size} of them, seed {report.seed}.",
344
+ f"- **Strict:** a real fix on the package's own repo: {_pct(report.real_fix_strict)}.",
345
+ f"- **Lenient:** also counting fixes whose package repo is not declared: "
346
+ f"{_pct(report.real_fix_lenient)}.",
347
+ "",
348
+ "Estimated share of *all* PyPI vulnerabilities with a real fix commit "
349
+ "(gate metric times sample share; the gate metric's own ±1.1-point interval is "
350
+ "ignored):",
351
+ "",
352
+ f"- Strict: **{_est(report.estimated_share_strict)}**",
353
+ f"- Lenient: {_est(report.estimated_share_lenient)}",
354
+ "",
355
+ "## Outcomes",
356
+ "",
357
+ "Each vulnerability gets the best outcome among its candidates.",
358
+ "",
359
+ "| Outcome | Vulnerabilities |",
360
+ "|---|---|",
361
+ *rows,
362
+ "",
363
+ f"Candidates checked: {report.candidates_checked} "
364
+ f"({', '.join(f'{k} {v}' for k, v in report.candidate_status.items())}).",
365
+ "",
366
+ "## What the real fixes look like",
367
+ "",
368
+ f"- Fixes that change Python source (what tree-sitter maps to functions): "
369
+ f"{_pct(report.fixes_touching_python)}. The rest change only native code "
370
+ "(C, Rust and so on) or templates and scripts, so they fall back to *unknown*.",
371
+ f"- Files changed per fix, including tests and docs: median "
372
+ f"{report.files_per_fix_median:.0f}, p90 {report.files_per_fix_p90:.0f}.",
373
+ f"- Fixes whose file list GitHub truncated: {report.truncated_fixes}.",
374
+ "",
375
+ "## Limits of this check",
376
+ "",
377
+ "- These are necessary conditions, not proof. A commit on the right repo that "
378
+ "changes source may still be a refactor rather than the fix; that is measured "
379
+ "against labelled data later.",
380
+ "- A package whose metadata names no GitHub repo and has no OSV GIT range lands "
381
+ "in *repo not declared*, even when the candidate is correct.",
382
+ "",
383
+ ]
384
+ )
@@ -0,0 +1,143 @@
1
+ """A JSON-over-HTTP client with an on-disk response cache, retries and typed errors.
2
+
3
+ Offline pipeline only. Responses, including "not found", are cached by request, so
4
+ re-runs make no network calls and give the same answers. The cache can hold third-party
5
+ code (diff patches), so it lives in an ignored directory and is never committed.
6
+ Credentials are sent as headers but never become part of a cache key.
7
+ """
8
+
9
+ import hashlib
10
+ import json
11
+ import time
12
+ from collections.abc import Callable, Mapping
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ import httpx
17
+ from tenacity import (
18
+ Retrying,
19
+ retry_if_exception_type,
20
+ stop_after_attempt,
21
+ wait_exponential,
22
+ )
23
+ from tenacity.wait import wait_base
24
+
25
+ NOT_FOUND_STATUSES = frozenset({404, 409, 410, 422, 451})
26
+ RETRY_STATUSES = frozenset({500, 502, 503, 504})
27
+
28
+
29
+ class HttpError(RuntimeError):
30
+ """A request failed in a way that retrying will not fix."""
31
+
32
+
33
+ class NotFoundError(HttpError):
34
+ """The resource does not exist, or never will for this request."""
35
+
36
+
37
+ class RateLimitError(HttpError):
38
+ """The server's rate limit is exhausted."""
39
+
40
+ def __init__(self, message: str, reset_at: float | None = None) -> None:
41
+ super().__init__(message)
42
+ self.reset_at = reset_at
43
+
44
+
45
+ class _RetryableError(Exception):
46
+ pass
47
+
48
+
49
+ class CachedJsonClient:
50
+ def __init__(
51
+ self,
52
+ base_url: str,
53
+ cache_dir: Path,
54
+ headers: Mapping[str, str] | None = None,
55
+ transport: httpx.BaseTransport | None = None,
56
+ max_attempts: int = 4,
57
+ wait: wait_base | None = None,
58
+ wait_for_rate_limit: float = 0.0,
59
+ sleep: Callable[[float], None] = time.sleep,
60
+ clock: Callable[[], float] = time.time,
61
+ ) -> None:
62
+ """``wait_for_rate_limit``: the longest wait, in seconds, for an exhausted rate
63
+ limit to reset before retrying. Zero raises ``RateLimitError`` at once."""
64
+ self._cache_dir = cache_dir
65
+ self._max_attempts = max_attempts
66
+ self._wait_for_rate_limit = wait_for_rate_limit
67
+ self._sleep = sleep
68
+ self._clock = clock
69
+ self.rate_limit_waits = 0
70
+ self._http = httpx.Client(
71
+ base_url=base_url,
72
+ headers=dict(headers or {}),
73
+ timeout=httpx.Timeout(30.0, connect=10.0),
74
+ follow_redirects=True,
75
+ transport=transport,
76
+ )
77
+ self._retrying: Callable[[], Retrying] = lambda: Retrying(
78
+ retry=retry_if_exception_type(_RetryableError),
79
+ stop=stop_after_attempt(max_attempts),
80
+ wait=wait or wait_exponential(multiplier=1, max=30),
81
+ reraise=True,
82
+ )
83
+ self.network_requests = 0
84
+
85
+ def close(self) -> None:
86
+ self._http.close()
87
+
88
+ def _cache_path(self, path: str, params: Mapping[str, str | int]) -> Path:
89
+ query = "&".join(f"{k}={params[k]}" for k in sorted(params))
90
+ key = hashlib.sha256(f"{self._http.base_url}{path}?{query}".encode()).hexdigest()
91
+ return self._cache_dir / key[:2] / f"{key}.json"
92
+
93
+ def get_json(self, path: str, params: Mapping[str, str | int] | None = None) -> Any:
94
+ params = dict(params or {})
95
+ cache_file = self._cache_path(path, params)
96
+ if cache_file.exists():
97
+ cached = json.loads(cache_file.read_text(encoding="utf-8"))
98
+ else:
99
+ cached = self._fetch(path, params)
100
+ cache_file.parent.mkdir(parents=True, exist_ok=True)
101
+ cache_file.write_text(json.dumps(cached), encoding="utf-8")
102
+ if cached["status"] != 200:
103
+ raise NotFoundError(f"{path}: HTTP {cached['status']}")
104
+ return cached["body"]
105
+
106
+ def _fetch(self, path: str, params: Mapping[str, str | int]) -> dict[str, Any]:
107
+ while True:
108
+ try:
109
+ for attempt in self._retrying():
110
+ with attempt:
111
+ return self._fetch_once(path, params)
112
+ except _RetryableError as error:
113
+ # Out of retries: an ordinary, catchable failure, not an internal type.
114
+ raise HttpError(f"gave up after {self._max_attempts} attempts: {error}") from None
115
+ except RateLimitError as limited:
116
+ if limited.reset_at is None:
117
+ raise
118
+ wait = limited.reset_at - self._clock() + 5.0
119
+ if not 0 < wait <= self._wait_for_rate_limit:
120
+ raise
121
+ self.rate_limit_waits += 1
122
+ self._sleep(wait)
123
+ else: # pragma: no cover - the loop above always returns or raises
124
+ raise AssertionError("unreachable")
125
+
126
+ def _fetch_once(self, path: str, params: Mapping[str, str | int]) -> dict[str, Any]:
127
+ self.network_requests += 1
128
+ try:
129
+ response = self._http.get(path, params=dict(params))
130
+ except httpx.TransportError as error:
131
+ raise _RetryableError(str(error)) from error
132
+ status = response.status_code
133
+ if status == 200:
134
+ return {"status": 200, "body": response.json()}
135
+ if status in NOT_FOUND_STATUSES:
136
+ return {"status": status, "body": None}
137
+ if status in (403, 429) and response.headers.get("x-ratelimit-remaining") == "0":
138
+ reset = response.headers.get("x-ratelimit-reset")
139
+ reset_at = float(reset) if reset and reset.isdigit() else None
140
+ raise RateLimitError(f"{path}: rate limit exhausted until {reset}", reset_at)
141
+ if status in RETRY_STATUSES or status == 429:
142
+ raise _RetryableError(f"{path}: HTTP {status}")
143
+ raise HttpError(f"{path}: HTTP {status}")
@@ -0,0 +1 @@
1
+ """Ingestion: source snapshots, OSV normalisation, fix-commit references, alias groups."""
@@ -0,0 +1,76 @@
1
+ """Group advisories that describe the same vulnerability.
2
+
3
+ The same vulnerability usually appears several times in the PyPI dump, for example as a
4
+ GHSA record and a PYSEC record that alias each other or share a CVE. Counting and
5
+ extraction work per group, so it is done once per vulnerability, not once per record.
6
+ """
7
+
8
+ from collections.abc import Iterable
9
+ from datetime import datetime
10
+
11
+ from pydantic import BaseModel, ConfigDict
12
+
13
+ from patchsight.pipeline.ingest.osv import Advisory
14
+
15
+
16
+ class AliasGroup(BaseModel):
17
+ model_config = ConfigDict(frozen=True)
18
+
19
+ key: str
20
+ advisory_ids: tuple[str, ...]
21
+ identifiers: tuple[str, ...]
22
+
23
+
24
+ class _UnionFind:
25
+ def __init__(self) -> None:
26
+ self.parent: dict[str, str] = {}
27
+
28
+ def find(self, item: str) -> str:
29
+ self.parent.setdefault(item, item)
30
+ root = item
31
+ while self.parent[root] != root:
32
+ root = self.parent[root]
33
+ while self.parent[item] != root:
34
+ self.parent[item], item = root, self.parent[item]
35
+ return root
36
+
37
+ def union(self, a: str, b: str) -> None:
38
+ root_a, root_b = self.find(a), self.find(b)
39
+ if root_a != root_b:
40
+ low, high = sorted((root_a, root_b))
41
+ self.parent[high] = low
42
+
43
+
44
+ def alias_groups(advisories: Iterable[Advisory]) -> list[AliasGroup]:
45
+ """Group advisories connected by any shared identifier. Independent of input order.
46
+
47
+ The group key is the smallest advisory ID in the group, so it is deterministic.
48
+ """
49
+ union = _UnionFind()
50
+ advisory_ids: set[str] = set()
51
+ for advisory in advisories:
52
+ advisory_ids.add(advisory.id)
53
+ union.find(advisory.id)
54
+ for alias in advisory.aliases:
55
+ union.union(advisory.id, alias)
56
+
57
+ members: dict[str, list[str]] = {}
58
+ for identifier in union.parent:
59
+ members.setdefault(union.find(identifier), []).append(identifier)
60
+
61
+ groups = []
62
+ for identifiers in members.values():
63
+ ids = sorted(i for i in identifiers if i in advisory_ids)
64
+ if ids:
65
+ groups.append(
66
+ AliasGroup(
67
+ key=ids[0], advisory_ids=tuple(ids), identifiers=tuple(sorted(identifiers))
68
+ )
69
+ )
70
+ return sorted(groups, key=lambda group: group.key)
71
+
72
+
73
+ def first_published(group: AliasGroup, by_id: dict[str, Advisory]) -> datetime | None:
74
+ """The earliest ``published`` timestamp among the group's advisories."""
75
+ dates = [d for i in group.advisory_ids if (d := by_id[i].published) is not None]
76
+ return min(dates) if dates else None
@@ -0,0 +1,80 @@
1
+ """Partial, sparse clone of ``github/advisory-database``, limited to PyPI advisories.
2
+
3
+ The full repository is several gigabytes. GHSA files live at
4
+ ``advisories/github-reviewed/<YYYY>/<MM>/<ID>/<ID>.json``, keyed by publication month, so
5
+ the PyPI subset can be checked out directly from the OSV records. Cone-mode sparse
6
+ checkout keeps this to seconds; non-cone patterns are matched against every path and are
7
+ orders of magnitude slower.
8
+ """
9
+
10
+ import json
11
+ import subprocess
12
+ from collections.abc import Iterable
13
+ from pathlib import Path
14
+
15
+ from pydantic import BaseModel, ConfigDict
16
+
17
+ from patchsight.pipeline.ingest.osv import Advisory
18
+
19
+ GHSA_REPO_URL = "https://github.com/github/advisory-database.git"
20
+
21
+
22
+ class GhsaScan(BaseModel):
23
+ model_config = ConfigDict(frozen=True)
24
+
25
+ expected: int
26
+ found: int
27
+ missing: tuple[str, ...]
28
+ with_symbol_data: tuple[str, ...]
29
+
30
+
31
+ def ghsa_directory(advisory: Advisory) -> str | None:
32
+ if advisory.source != "GHSA" or advisory.published is None:
33
+ return None
34
+ month = advisory.published.strftime("%Y/%m")
35
+ return f"advisories/github-reviewed/{month}/{advisory.id}"
36
+
37
+
38
+ def _git(*args: str) -> None:
39
+ subprocess.run(["git", *args], check=True) # noqa: S603, S607 - fixed argv, no shell
40
+
41
+
42
+ def sparse_clone(dest: Path, advisories: Iterable[Advisory]) -> None:
43
+ """Clone only the directories of the given GHSA advisories into ``dest``."""
44
+ directories = sorted({d for a in advisories if (d := ghsa_directory(a))})
45
+ if not (dest / ".git").exists():
46
+ _git("clone", "-q", "--depth", "1", "--filter=blob:none", "--no-checkout",
47
+ GHSA_REPO_URL, str(dest)) # fmt: skip
48
+ subprocess.run( # noqa: S603 - fixed argv, no shell
49
+ ["git", "-C", str(dest), "sparse-checkout", "set", "--cone", "--stdin"], # noqa: S607
50
+ input="\n".join(directories),
51
+ text=True,
52
+ check=True,
53
+ )
54
+ _git("-C", str(dest), "checkout", "-q", "main")
55
+
56
+
57
+ def scan(repo: Path, advisories: Iterable[Advisory]) -> GhsaScan:
58
+ """Check which GHSA advisories in the clone carry ``ecosystem_specific`` symbol data."""
59
+ expected, missing, with_symbols = 0, [], []
60
+ for advisory in advisories:
61
+ directory = ghsa_directory(advisory)
62
+ if directory is None:
63
+ continue
64
+ expected += 1
65
+ path = repo / directory / f"{advisory.id}.json"
66
+ if not path.exists():
67
+ missing.append(advisory.id)
68
+ continue
69
+ record = json.loads(path.read_text(encoding="utf-8"))
70
+ for affected in record.get("affected", []):
71
+ specific = affected.get("ecosystem_specific") or {}
72
+ if specific.get("affected_functions") or specific.get("imports"):
73
+ with_symbols.append(advisory.id)
74
+ break
75
+ return GhsaScan(
76
+ expected=expected,
77
+ found=expected - len(missing),
78
+ missing=tuple(sorted(missing)),
79
+ with_symbol_data=tuple(sorted(with_symbols)),
80
+ )