patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Read the OSV PyPI dump and normalise each record.
|
|
2
|
+
|
|
3
|
+
Every record ends in exactly one of three states:
|
|
4
|
+
|
|
5
|
+
* ``Advisory``: an in-scope PyPI vulnerability, normalised.
|
|
6
|
+
* ``Excluded``: valid but out of scope, with the reason (withdrawn, malicious-package
|
|
7
|
+
report, no PyPI package).
|
|
8
|
+
* ``Quarantined``: malformed, with the validation error. Never silently skipped.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import zipfile
|
|
13
|
+
from collections.abc import Iterator
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Literal
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
|
19
|
+
|
|
20
|
+
from patchsight.names import canonical_package_name
|
|
21
|
+
from patchsight.pipeline.ingest.references import (
|
|
22
|
+
FixCandidate,
|
|
23
|
+
candidate_from_git_range,
|
|
24
|
+
classify_url,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# --- Raw OSV subset (only the fields PatchSight reads; others are ignored) -------------
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _Raw(BaseModel):
|
|
31
|
+
model_config = ConfigDict(extra="ignore", frozen=True)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class OsvReference(_Raw):
|
|
35
|
+
type: str
|
|
36
|
+
url: str
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class OsvEvent(_Raw):
|
|
40
|
+
introduced: str | None = None
|
|
41
|
+
fixed: str | None = None
|
|
42
|
+
last_affected: str | None = None
|
|
43
|
+
limit: str | None = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class OsvRange(_Raw):
|
|
47
|
+
type: str
|
|
48
|
+
repo: str | None = None
|
|
49
|
+
events: tuple[OsvEvent, ...] = ()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class OsvPackage(_Raw):
|
|
53
|
+
ecosystem: str
|
|
54
|
+
name: str
|
|
55
|
+
purl: str | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class OsvAffected(_Raw):
|
|
59
|
+
package: OsvPackage
|
|
60
|
+
ranges: tuple[OsvRange, ...] = ()
|
|
61
|
+
versions: tuple[str, ...] = ()
|
|
62
|
+
ecosystem_specific: dict[str, Any] | None = None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class OsvRecord(_Raw):
|
|
66
|
+
id: str = Field(min_length=1)
|
|
67
|
+
modified: datetime
|
|
68
|
+
published: datetime | None = None
|
|
69
|
+
withdrawn: datetime | None = None
|
|
70
|
+
aliases: tuple[str, ...] = ()
|
|
71
|
+
related: tuple[str, ...] = ()
|
|
72
|
+
summary: str = ""
|
|
73
|
+
details: str = ""
|
|
74
|
+
affected: tuple[OsvAffected, ...] = ()
|
|
75
|
+
references: tuple[OsvReference, ...] = ()
|
|
76
|
+
schema_version: str | None = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# --- Normalised output -----------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class VersionRange(BaseModel):
|
|
83
|
+
model_config = ConfigDict(frozen=True)
|
|
84
|
+
|
|
85
|
+
type: str
|
|
86
|
+
events: tuple[OsvEvent, ...]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class SymbolHint(BaseModel):
|
|
90
|
+
"""Symbol data an upstream database already attaches to an advisory."""
|
|
91
|
+
|
|
92
|
+
model_config = ConfigDict(frozen=True)
|
|
93
|
+
|
|
94
|
+
field: Literal["imports", "affected_functions"]
|
|
95
|
+
package: str
|
|
96
|
+
value: Any
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class AffectedPackage(BaseModel):
|
|
100
|
+
model_config = ConfigDict(frozen=True)
|
|
101
|
+
|
|
102
|
+
name: str
|
|
103
|
+
raw_name: str
|
|
104
|
+
ranges: tuple[VersionRange, ...]
|
|
105
|
+
versions: tuple[str, ...]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class Advisory(BaseModel):
|
|
109
|
+
model_config = ConfigDict(frozen=True)
|
|
110
|
+
|
|
111
|
+
id: str
|
|
112
|
+
source: str
|
|
113
|
+
aliases: tuple[str, ...]
|
|
114
|
+
related: tuple[str, ...]
|
|
115
|
+
published: datetime | None
|
|
116
|
+
modified: datetime
|
|
117
|
+
summary: str
|
|
118
|
+
details: str
|
|
119
|
+
packages: tuple[AffectedPackage, ...]
|
|
120
|
+
references: tuple[OsvReference, ...]
|
|
121
|
+
fix_candidates: tuple[FixCandidate, ...]
|
|
122
|
+
symbol_hints: tuple[SymbolHint, ...]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
ExclusionReason = Literal["withdrawn", "malicious_package", "no_pypi_package"]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class Excluded(BaseModel):
|
|
129
|
+
model_config = ConfigDict(frozen=True)
|
|
130
|
+
|
|
131
|
+
id: str
|
|
132
|
+
reason: ExclusionReason
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class Quarantined(BaseModel):
|
|
136
|
+
model_config = ConfigDict(frozen=True)
|
|
137
|
+
|
|
138
|
+
source_name: str
|
|
139
|
+
reason: str
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
NormaliseResult = Advisory | Excluded | Quarantined
|
|
143
|
+
|
|
144
|
+
_SYMBOL_FIELDS: tuple[Literal["imports", "affected_functions"], ...] = (
|
|
145
|
+
"imports",
|
|
146
|
+
"affected_functions",
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def source_of(advisory_id: str) -> str:
|
|
151
|
+
"""The database prefix of an advisory ID, e.g. ``GHSA`` or ``PYSEC``."""
|
|
152
|
+
return advisory_id.split("-", 1)[0].upper()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _fix_candidates(record: OsvRecord) -> tuple[FixCandidate, ...]:
|
|
156
|
+
found: dict[tuple[str, str, str | None, str | None, int | None], FixCandidate] = {}
|
|
157
|
+
for affected in record.affected:
|
|
158
|
+
for version_range in affected.ranges:
|
|
159
|
+
if version_range.type != "GIT" or not version_range.repo:
|
|
160
|
+
continue
|
|
161
|
+
for event in version_range.events:
|
|
162
|
+
if event.fixed and (c := candidate_from_git_range(version_range.repo, event.fixed)):
|
|
163
|
+
found.setdefault((c.origin, c.kind, c.repo, c.commit, c.number), c)
|
|
164
|
+
for reference in record.references:
|
|
165
|
+
if c := classify_url(reference.url, reference.type):
|
|
166
|
+
found.setdefault((c.origin, c.kind, c.repo, c.commit or c.url, c.number), c)
|
|
167
|
+
return tuple(found.values())
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _symbol_hints(record: OsvRecord) -> tuple[SymbolHint, ...]:
|
|
171
|
+
hints = []
|
|
172
|
+
for affected in record.affected:
|
|
173
|
+
specific = affected.ecosystem_specific or {}
|
|
174
|
+
for field in _SYMBOL_FIELDS:
|
|
175
|
+
if specific.get(field):
|
|
176
|
+
hints.append(
|
|
177
|
+
SymbolHint(
|
|
178
|
+
field=field,
|
|
179
|
+
package=canonical_package_name(affected.package.name),
|
|
180
|
+
value=specific[field],
|
|
181
|
+
)
|
|
182
|
+
)
|
|
183
|
+
return tuple(hints)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def normalise(raw: dict[str, Any], source_name: str) -> NormaliseResult:
|
|
187
|
+
"""Normalise one raw OSV record. Never raises on bad input; quarantines it instead."""
|
|
188
|
+
try:
|
|
189
|
+
record = OsvRecord.model_validate(raw)
|
|
190
|
+
except ValidationError as error:
|
|
191
|
+
first = error.errors()[0]
|
|
192
|
+
location = ".".join(str(part) for part in first["loc"])
|
|
193
|
+
return Quarantined(source_name=source_name, reason=f"{location}: {first['msg']}")
|
|
194
|
+
|
|
195
|
+
if record.withdrawn is not None:
|
|
196
|
+
return Excluded(id=record.id, reason="withdrawn")
|
|
197
|
+
if source_of(record.id) == "MAL":
|
|
198
|
+
return Excluded(id=record.id, reason="malicious_package")
|
|
199
|
+
pypi = [a for a in record.affected if a.package.ecosystem == "PyPI"]
|
|
200
|
+
if not pypi:
|
|
201
|
+
return Excluded(id=record.id, reason="no_pypi_package")
|
|
202
|
+
|
|
203
|
+
packages = tuple(
|
|
204
|
+
AffectedPackage(
|
|
205
|
+
name=canonical_package_name(a.package.name),
|
|
206
|
+
raw_name=a.package.name,
|
|
207
|
+
ranges=tuple(
|
|
208
|
+
VersionRange(type=r.type, events=r.events) for r in a.ranges if r.type != "GIT"
|
|
209
|
+
),
|
|
210
|
+
versions=a.versions,
|
|
211
|
+
)
|
|
212
|
+
for a in pypi
|
|
213
|
+
)
|
|
214
|
+
return Advisory(
|
|
215
|
+
id=record.id,
|
|
216
|
+
source=source_of(record.id),
|
|
217
|
+
aliases=tuple(sorted(set(record.aliases))),
|
|
218
|
+
related=tuple(sorted(set(record.related))),
|
|
219
|
+
published=record.published,
|
|
220
|
+
modified=record.modified,
|
|
221
|
+
summary=record.summary,
|
|
222
|
+
details=record.details,
|
|
223
|
+
packages=packages,
|
|
224
|
+
references=record.references,
|
|
225
|
+
fix_candidates=_fix_candidates(record),
|
|
226
|
+
symbol_hints=_symbol_hints(record),
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def iter_dump(zip_path: Path) -> Iterator[tuple[str, dict[str, Any] | None, str | None]]:
|
|
231
|
+
"""Yield ``(member name, parsed JSON or None, parse error or None)`` in name order."""
|
|
232
|
+
with zipfile.ZipFile(zip_path) as archive:
|
|
233
|
+
for name in sorted(archive.namelist()):
|
|
234
|
+
if name.endswith("/"):
|
|
235
|
+
continue
|
|
236
|
+
try:
|
|
237
|
+
data = json.loads(archive.read(name))
|
|
238
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
|
239
|
+
yield name, None, f"invalid JSON: {error}"
|
|
240
|
+
continue
|
|
241
|
+
if not isinstance(data, dict):
|
|
242
|
+
yield name, None, "top-level JSON value is not an object"
|
|
243
|
+
continue
|
|
244
|
+
yield name, data, None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def normalise_dump(zip_path: Path) -> Iterator[NormaliseResult]:
|
|
248
|
+
"""Normalise every record in an OSV dump, in deterministic order."""
|
|
249
|
+
for name, data, error in iter_dump(zip_path):
|
|
250
|
+
if data is None:
|
|
251
|
+
yield Quarantined(source_name=name, reason=error or "unreadable")
|
|
252
|
+
else:
|
|
253
|
+
yield normalise(data, name)
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Classify advisory reference URLs and git ranges into candidate fix commits.
|
|
2
|
+
|
|
3
|
+
A candidate is a pointer to code that may contain the fix: a commit, a pull request or a
|
|
4
|
+
tag comparison. Whether it is on the package's own repository, and whether it really is
|
|
5
|
+
the fix, is decided later, when the diff is fetched.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from typing import Literal
|
|
10
|
+
from urllib.parse import parse_qs, urlsplit
|
|
11
|
+
|
|
12
|
+
from pydantic import BaseModel, ConfigDict
|
|
13
|
+
|
|
14
|
+
CandidateKind = Literal[
|
|
15
|
+
"github_commit",
|
|
16
|
+
"github_pull",
|
|
17
|
+
"github_compare",
|
|
18
|
+
"gitlab_commit",
|
|
19
|
+
"gitlab_merge_request",
|
|
20
|
+
"bitbucket_commit",
|
|
21
|
+
"other_commit",
|
|
22
|
+
]
|
|
23
|
+
CandidateOrigin = Literal["reference", "git_range"]
|
|
24
|
+
|
|
25
|
+
_SHA = r"[0-9a-fA-F]{7,40}"
|
|
26
|
+
_END = r"(?:[/.#?]|$)"
|
|
27
|
+
_GH_REPO = r"^/(?P<owner>[A-Za-z0-9_.-]+)/(?P<repo>[A-Za-z0-9_.-]+)"
|
|
28
|
+
_GH_PR_COMMIT = re.compile(_GH_REPO + rf"/pull/\d+/commits/(?P<sha>{_SHA}){_END}")
|
|
29
|
+
_GH_COMMIT = re.compile(_GH_REPO + rf"/commits?/(?P<sha>{_SHA}){_END}")
|
|
30
|
+
_GH_PULL = re.compile(_GH_REPO + r"/pull/(?P<number>\d+)(?:[/#?]|$)")
|
|
31
|
+
_GH_COMPARE = re.compile(_GH_REPO + r"/compare/(?P<base>[^/?#]+?)\.{2,3}(?P<head>[^/?#]+)")
|
|
32
|
+
_GL_COMMIT = re.compile(rf"^/(?P<repo>.+?)/(?:-/)?commit/(?P<sha>{_SHA}){_END}")
|
|
33
|
+
_GL_MR = re.compile(r"^/(?P<repo>.+?)/(?:-/)?merge_requests/(?P<number>\d+)(?:[/#?]|$)")
|
|
34
|
+
_BB_COMMIT = re.compile(rf"^/(?P<owner>[^/]+)/(?P<repo>[^/]+)/commits?/(?P<sha>{_SHA}){_END}")
|
|
35
|
+
_ANY_COMMIT_PATH = re.compile(rf"/(?:-/)?commits?/(?P<sha>{_SHA}){_END}")
|
|
36
|
+
_GITWEB_HASH = re.compile(r";h=(?P<sha>[0-9a-fA-F]{40})")
|
|
37
|
+
_FULL_SHA = re.compile(r"^[0-9a-fA-F]{40}$")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class FixCandidate(BaseModel):
|
|
41
|
+
"""A pointer to code that may contain the fix for an advisory."""
|
|
42
|
+
|
|
43
|
+
model_config = ConfigDict(frozen=True)
|
|
44
|
+
|
|
45
|
+
kind: CandidateKind
|
|
46
|
+
origin: CandidateOrigin
|
|
47
|
+
url: str
|
|
48
|
+
host: str
|
|
49
|
+
repo: str | None = None
|
|
50
|
+
commit: str | None = None
|
|
51
|
+
number: int | None = None
|
|
52
|
+
base: str | None = None
|
|
53
|
+
head: str | None = None
|
|
54
|
+
reference_type: str | None = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _clean_repo(name: str) -> str:
|
|
58
|
+
return name.removesuffix(".git").lower()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _github(path: str) -> dict[str, object] | None:
|
|
62
|
+
for kind, pattern in (
|
|
63
|
+
("github_commit", _GH_PR_COMMIT),
|
|
64
|
+
("github_commit", _GH_COMMIT),
|
|
65
|
+
("github_pull", _GH_PULL),
|
|
66
|
+
("github_compare", _GH_COMPARE),
|
|
67
|
+
):
|
|
68
|
+
if match := pattern.match(path):
|
|
69
|
+
fields = match.groupdict()
|
|
70
|
+
result: dict[str, object] = {
|
|
71
|
+
"kind": kind,
|
|
72
|
+
"repo": f"{fields['owner'].lower()}/{_clean_repo(fields['repo'])}",
|
|
73
|
+
}
|
|
74
|
+
if sha := fields.get("sha"):
|
|
75
|
+
result["commit"] = sha.lower()
|
|
76
|
+
if number := fields.get("number"):
|
|
77
|
+
result["number"] = int(number)
|
|
78
|
+
if base := fields.get("base"):
|
|
79
|
+
result["base"], result["head"] = base, fields["head"]
|
|
80
|
+
return result
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _gitlab(path: str) -> dict[str, object] | None:
|
|
85
|
+
if match := _GL_COMMIT.match(path):
|
|
86
|
+
return {
|
|
87
|
+
"kind": "gitlab_commit",
|
|
88
|
+
"repo": _clean_repo(match["repo"]),
|
|
89
|
+
"commit": match["sha"].lower(),
|
|
90
|
+
}
|
|
91
|
+
if match := _GL_MR.match(path):
|
|
92
|
+
return {
|
|
93
|
+
"kind": "gitlab_merge_request",
|
|
94
|
+
"repo": _clean_repo(match["repo"]),
|
|
95
|
+
"number": int(match["number"]),
|
|
96
|
+
}
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _bitbucket(path: str) -> dict[str, object] | None:
|
|
101
|
+
if match := _BB_COMMIT.match(path):
|
|
102
|
+
return {
|
|
103
|
+
"kind": "bitbucket_commit",
|
|
104
|
+
"repo": f"{match['owner'].lower()}/{_clean_repo(match['repo'])}",
|
|
105
|
+
"commit": match["sha"].lower(),
|
|
106
|
+
}
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _self_hosted(url: str, path: str, query: str, host: str) -> dict[str, object] | None:
|
|
111
|
+
"""Self-hosted forges: ``/commit/<sha>`` paths, cgit ``?id=<sha>``, gitweb ``;h=<sha>``."""
|
|
112
|
+
sha: str | None = None
|
|
113
|
+
if match := _ANY_COMMIT_PATH.search(path):
|
|
114
|
+
sha = match["sha"]
|
|
115
|
+
else:
|
|
116
|
+
params = parse_qs(query)
|
|
117
|
+
for key in ("id", "h"):
|
|
118
|
+
values = params.get(key, [])
|
|
119
|
+
if values and _FULL_SHA.match(values[0]):
|
|
120
|
+
sha = values[0]
|
|
121
|
+
break
|
|
122
|
+
if sha is None and (match := _GITWEB_HASH.search(url)):
|
|
123
|
+
sha = match["sha"]
|
|
124
|
+
if sha is None or ("commit" not in url and "git" not in host):
|
|
125
|
+
return None
|
|
126
|
+
return {"kind": "other_commit", "commit": sha.lower()}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def classify_url(url: str, reference_type: str | None = None) -> FixCandidate | None:
|
|
130
|
+
"""Return a fix candidate if ``url`` points at a commit, pull request or comparison."""
|
|
131
|
+
try:
|
|
132
|
+
parts = urlsplit(url.strip())
|
|
133
|
+
host = (parts.hostname or "").lower()
|
|
134
|
+
except ValueError:
|
|
135
|
+
return None
|
|
136
|
+
if parts.scheme not in ("http", "https") or not host:
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
if host in ("github.com", "www.github.com"):
|
|
140
|
+
fields = _github(parts.path)
|
|
141
|
+
elif host == "gitlab.com" or host.startswith("gitlab."):
|
|
142
|
+
fields = _gitlab(parts.path)
|
|
143
|
+
elif host == "bitbucket.org":
|
|
144
|
+
fields = _bitbucket(parts.path)
|
|
145
|
+
else:
|
|
146
|
+
fields = _self_hosted(url, parts.path, parts.query, host)
|
|
147
|
+
if fields is None:
|
|
148
|
+
return None
|
|
149
|
+
return FixCandidate.model_validate(
|
|
150
|
+
{
|
|
151
|
+
**fields,
|
|
152
|
+
"origin": "reference",
|
|
153
|
+
"url": url,
|
|
154
|
+
"host": host,
|
|
155
|
+
"reference_type": reference_type,
|
|
156
|
+
}
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def candidate_from_git_range(repo_url: str, fixed: str) -> FixCandidate | None:
|
|
161
|
+
"""Turn an OSV ``GIT`` range ``fixed`` event into a fix candidate."""
|
|
162
|
+
if not _FULL_SHA.match(fixed):
|
|
163
|
+
return None
|
|
164
|
+
try:
|
|
165
|
+
parts = urlsplit(repo_url.strip())
|
|
166
|
+
host = (parts.hostname or "").lower()
|
|
167
|
+
except ValueError:
|
|
168
|
+
return None
|
|
169
|
+
if not host:
|
|
170
|
+
return None
|
|
171
|
+
path = parts.path.strip("/")
|
|
172
|
+
segments = path.split("/")
|
|
173
|
+
if host in ("github.com", "www.github.com", "bitbucket.org") and len(segments) >= 2:
|
|
174
|
+
kind = "bitbucket_commit" if host == "bitbucket.org" else "github_commit"
|
|
175
|
+
repo: str | None = f"{segments[0].lower()}/{_clean_repo(segments[1])}"
|
|
176
|
+
elif host == "gitlab.com" or host.startswith("gitlab."):
|
|
177
|
+
kind, repo = "gitlab_commit", _clean_repo(path)
|
|
178
|
+
else:
|
|
179
|
+
kind, repo = "other_commit", (_clean_repo(path) or None)
|
|
180
|
+
return FixCandidate.model_validate(
|
|
181
|
+
{
|
|
182
|
+
"kind": kind,
|
|
183
|
+
"origin": "git_range",
|
|
184
|
+
"url": repo_url,
|
|
185
|
+
"host": host,
|
|
186
|
+
"repo": repo,
|
|
187
|
+
"commit": fixed.lower(),
|
|
188
|
+
}
|
|
189
|
+
)
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Pinned source snapshots.
|
|
2
|
+
|
|
3
|
+
Every run of the pipeline reads inputs recorded in ``manifests/snapshot.json``: each
|
|
4
|
+
downloaded file with its SHA-256, and each cloned repository with its commit. A file whose
|
|
5
|
+
hash no longer matches the manifest is refused, so results are always traceable to the
|
|
6
|
+
exact inputs that produced them.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import subprocess
|
|
11
|
+
from datetime import UTC, datetime
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
from pydantic import BaseModel, ConfigDict
|
|
16
|
+
|
|
17
|
+
OSV_PYPI_DUMP_URL = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
|
|
18
|
+
OSV_PYPI_MODIFIED_URL = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/modified_id.csv"
|
|
19
|
+
|
|
20
|
+
_CHUNK = 1 << 20
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SnapshotMismatchError(RuntimeError):
|
|
24
|
+
"""A local file does not match the hash pinned in the snapshot manifest."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SourceFile(BaseModel):
|
|
28
|
+
model_config = ConfigDict(frozen=True)
|
|
29
|
+
|
|
30
|
+
name: str
|
|
31
|
+
url: str
|
|
32
|
+
path: str
|
|
33
|
+
sha256: str
|
|
34
|
+
size: int
|
|
35
|
+
last_modified: str | None
|
|
36
|
+
retrieved_at: datetime
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class SourceRepo(BaseModel):
|
|
40
|
+
model_config = ConfigDict(frozen=True)
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
url: str
|
|
44
|
+
path: str
|
|
45
|
+
commit: str
|
|
46
|
+
committed_at: datetime
|
|
47
|
+
note: str = ""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class SnapshotManifest(BaseModel):
|
|
51
|
+
files: tuple[SourceFile, ...] = ()
|
|
52
|
+
repos: tuple[SourceRepo, ...] = ()
|
|
53
|
+
|
|
54
|
+
def file(self, name: str) -> SourceFile:
|
|
55
|
+
for entry in self.files:
|
|
56
|
+
if entry.name == name:
|
|
57
|
+
return entry
|
|
58
|
+
raise KeyError(f"no file named {name!r} in the snapshot manifest")
|
|
59
|
+
|
|
60
|
+
def with_file(self, entry: SourceFile) -> "SnapshotManifest":
|
|
61
|
+
others = tuple(f for f in self.files if f.name != entry.name)
|
|
62
|
+
return self.model_copy(update={"files": tuple(sorted((*others, entry), key=_name))})
|
|
63
|
+
|
|
64
|
+
def with_repo(self, entry: SourceRepo) -> "SnapshotManifest":
|
|
65
|
+
others = tuple(r for r in self.repos if r.name != entry.name)
|
|
66
|
+
return self.model_copy(update={"repos": tuple(sorted((*others, entry), key=_name))})
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _name(entry: SourceFile | SourceRepo) -> str:
|
|
70
|
+
return entry.name
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def sha256_file(path: Path) -> str:
|
|
74
|
+
digest = hashlib.sha256()
|
|
75
|
+
with path.open("rb") as handle:
|
|
76
|
+
while chunk := handle.read(_CHUNK):
|
|
77
|
+
digest.update(chunk)
|
|
78
|
+
return digest.hexdigest()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def load_manifest(path: Path) -> SnapshotManifest:
|
|
82
|
+
if not path.exists():
|
|
83
|
+
return SnapshotManifest()
|
|
84
|
+
return SnapshotManifest.model_validate_json(path.read_text(encoding="utf-8"))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def save_manifest(manifest: SnapshotManifest, path: Path) -> None:
|
|
88
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
89
|
+
path.write_text(manifest.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def download(client: httpx.Client, name: str, url: str, dest: Path, root: Path) -> SourceFile:
|
|
93
|
+
"""Stream ``url`` to ``dest``, hashing as it goes. Replaces ``dest`` only on success."""
|
|
94
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
95
|
+
partial = dest.with_name(dest.name + ".part")
|
|
96
|
+
digest = hashlib.sha256()
|
|
97
|
+
size = 0
|
|
98
|
+
with client.stream("GET", url) as response:
|
|
99
|
+
response.raise_for_status()
|
|
100
|
+
last_modified = response.headers.get("last-modified")
|
|
101
|
+
with partial.open("wb") as handle:
|
|
102
|
+
for chunk in response.iter_bytes(_CHUNK):
|
|
103
|
+
handle.write(chunk)
|
|
104
|
+
digest.update(chunk)
|
|
105
|
+
size += len(chunk)
|
|
106
|
+
partial.replace(dest)
|
|
107
|
+
return SourceFile(
|
|
108
|
+
name=name,
|
|
109
|
+
url=url,
|
|
110
|
+
path=dest.relative_to(root).as_posix(),
|
|
111
|
+
sha256=digest.hexdigest(),
|
|
112
|
+
size=size,
|
|
113
|
+
last_modified=last_modified,
|
|
114
|
+
retrieved_at=datetime.now(UTC),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def verify(entry: SourceFile, root: Path) -> Path:
|
|
119
|
+
"""Return the local path of ``entry`` after checking its hash against the manifest."""
|
|
120
|
+
path = root / entry.path
|
|
121
|
+
if not path.exists():
|
|
122
|
+
raise SnapshotMismatchError(f"{entry.path} is missing; run the snapshot step first")
|
|
123
|
+
actual = sha256_file(path)
|
|
124
|
+
if actual != entry.sha256:
|
|
125
|
+
raise SnapshotMismatchError(
|
|
126
|
+
f"{entry.path} has SHA-256 {actual}, but the manifest pins {entry.sha256}"
|
|
127
|
+
)
|
|
128
|
+
return path
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def record_repo(name: str, url: str, path: Path, root: Path, note: str = "") -> SourceRepo:
|
|
132
|
+
"""Record the checked-out commit of a local clone."""
|
|
133
|
+
output = subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
134
|
+
["git", "-C", str(path), "log", "-1", "--format=%H %cI"], # noqa: S607
|
|
135
|
+
capture_output=True,
|
|
136
|
+
text=True,
|
|
137
|
+
check=True,
|
|
138
|
+
).stdout.split()
|
|
139
|
+
return SourceRepo(
|
|
140
|
+
name=name,
|
|
141
|
+
url=url,
|
|
142
|
+
path=path.relative_to(root).as_posix(),
|
|
143
|
+
commit=output[0],
|
|
144
|
+
committed_at=datetime.fromisoformat(output[1]),
|
|
145
|
+
note=note,
|
|
146
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""LLM access for the offline pipeline: one OpenAI-compatible client, cached and rate-limited."""
|