patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,253 @@
1
+ """Read the OSV PyPI dump and normalise each record.
2
+
3
+ Every record ends in exactly one of three states:
4
+
5
+ * ``Advisory``: an in-scope PyPI vulnerability, normalised.
6
+ * ``Excluded``: valid but out of scope, with the reason (withdrawn, malicious-package
7
+ report, no PyPI package).
8
+ * ``Quarantined``: malformed, with the validation error. Never silently skipped.
9
+ """
10
+
11
+ import json
12
+ import zipfile
13
+ from collections.abc import Iterator
14
+ from datetime import datetime
15
+ from pathlib import Path
16
+ from typing import Any, Literal
17
+
18
+ from pydantic import BaseModel, ConfigDict, Field, ValidationError
19
+
20
+ from patchsight.names import canonical_package_name
21
+ from patchsight.pipeline.ingest.references import (
22
+ FixCandidate,
23
+ candidate_from_git_range,
24
+ classify_url,
25
+ )
26
+
27
+ # --- Raw OSV subset (only the fields PatchSight reads; others are ignored) -------------
28
+
29
+
30
+ class _Raw(BaseModel):
31
+ model_config = ConfigDict(extra="ignore", frozen=True)
32
+
33
+
34
+ class OsvReference(_Raw):
35
+ type: str
36
+ url: str
37
+
38
+
39
+ class OsvEvent(_Raw):
40
+ introduced: str | None = None
41
+ fixed: str | None = None
42
+ last_affected: str | None = None
43
+ limit: str | None = None
44
+
45
+
46
+ class OsvRange(_Raw):
47
+ type: str
48
+ repo: str | None = None
49
+ events: tuple[OsvEvent, ...] = ()
50
+
51
+
52
+ class OsvPackage(_Raw):
53
+ ecosystem: str
54
+ name: str
55
+ purl: str | None = None
56
+
57
+
58
+ class OsvAffected(_Raw):
59
+ package: OsvPackage
60
+ ranges: tuple[OsvRange, ...] = ()
61
+ versions: tuple[str, ...] = ()
62
+ ecosystem_specific: dict[str, Any] | None = None
63
+
64
+
65
+ class OsvRecord(_Raw):
66
+ id: str = Field(min_length=1)
67
+ modified: datetime
68
+ published: datetime | None = None
69
+ withdrawn: datetime | None = None
70
+ aliases: tuple[str, ...] = ()
71
+ related: tuple[str, ...] = ()
72
+ summary: str = ""
73
+ details: str = ""
74
+ affected: tuple[OsvAffected, ...] = ()
75
+ references: tuple[OsvReference, ...] = ()
76
+ schema_version: str | None = None
77
+
78
+
79
+ # --- Normalised output -----------------------------------------------------------------
80
+
81
+
82
+ class VersionRange(BaseModel):
83
+ model_config = ConfigDict(frozen=True)
84
+
85
+ type: str
86
+ events: tuple[OsvEvent, ...]
87
+
88
+
89
+ class SymbolHint(BaseModel):
90
+ """Symbol data an upstream database already attaches to an advisory."""
91
+
92
+ model_config = ConfigDict(frozen=True)
93
+
94
+ field: Literal["imports", "affected_functions"]
95
+ package: str
96
+ value: Any
97
+
98
+
99
+ class AffectedPackage(BaseModel):
100
+ model_config = ConfigDict(frozen=True)
101
+
102
+ name: str
103
+ raw_name: str
104
+ ranges: tuple[VersionRange, ...]
105
+ versions: tuple[str, ...]
106
+
107
+
108
+ class Advisory(BaseModel):
109
+ model_config = ConfigDict(frozen=True)
110
+
111
+ id: str
112
+ source: str
113
+ aliases: tuple[str, ...]
114
+ related: tuple[str, ...]
115
+ published: datetime | None
116
+ modified: datetime
117
+ summary: str
118
+ details: str
119
+ packages: tuple[AffectedPackage, ...]
120
+ references: tuple[OsvReference, ...]
121
+ fix_candidates: tuple[FixCandidate, ...]
122
+ symbol_hints: tuple[SymbolHint, ...]
123
+
124
+
125
+ ExclusionReason = Literal["withdrawn", "malicious_package", "no_pypi_package"]
126
+
127
+
128
+ class Excluded(BaseModel):
129
+ model_config = ConfigDict(frozen=True)
130
+
131
+ id: str
132
+ reason: ExclusionReason
133
+
134
+
135
+ class Quarantined(BaseModel):
136
+ model_config = ConfigDict(frozen=True)
137
+
138
+ source_name: str
139
+ reason: str
140
+
141
+
142
+ NormaliseResult = Advisory | Excluded | Quarantined
143
+
144
+ _SYMBOL_FIELDS: tuple[Literal["imports", "affected_functions"], ...] = (
145
+ "imports",
146
+ "affected_functions",
147
+ )
148
+
149
+
150
+ def source_of(advisory_id: str) -> str:
151
+ """The database prefix of an advisory ID, e.g. ``GHSA`` or ``PYSEC``."""
152
+ return advisory_id.split("-", 1)[0].upper()
153
+
154
+
155
+ def _fix_candidates(record: OsvRecord) -> tuple[FixCandidate, ...]:
156
+ found: dict[tuple[str, str, str | None, str | None, int | None], FixCandidate] = {}
157
+ for affected in record.affected:
158
+ for version_range in affected.ranges:
159
+ if version_range.type != "GIT" or not version_range.repo:
160
+ continue
161
+ for event in version_range.events:
162
+ if event.fixed and (c := candidate_from_git_range(version_range.repo, event.fixed)):
163
+ found.setdefault((c.origin, c.kind, c.repo, c.commit, c.number), c)
164
+ for reference in record.references:
165
+ if c := classify_url(reference.url, reference.type):
166
+ found.setdefault((c.origin, c.kind, c.repo, c.commit or c.url, c.number), c)
167
+ return tuple(found.values())
168
+
169
+
170
+ def _symbol_hints(record: OsvRecord) -> tuple[SymbolHint, ...]:
171
+ hints = []
172
+ for affected in record.affected:
173
+ specific = affected.ecosystem_specific or {}
174
+ for field in _SYMBOL_FIELDS:
175
+ if specific.get(field):
176
+ hints.append(
177
+ SymbolHint(
178
+ field=field,
179
+ package=canonical_package_name(affected.package.name),
180
+ value=specific[field],
181
+ )
182
+ )
183
+ return tuple(hints)
184
+
185
+
186
+ def normalise(raw: dict[str, Any], source_name: str) -> NormaliseResult:
187
+ """Normalise one raw OSV record. Never raises on bad input; quarantines it instead."""
188
+ try:
189
+ record = OsvRecord.model_validate(raw)
190
+ except ValidationError as error:
191
+ first = error.errors()[0]
192
+ location = ".".join(str(part) for part in first["loc"])
193
+ return Quarantined(source_name=source_name, reason=f"{location}: {first['msg']}")
194
+
195
+ if record.withdrawn is not None:
196
+ return Excluded(id=record.id, reason="withdrawn")
197
+ if source_of(record.id) == "MAL":
198
+ return Excluded(id=record.id, reason="malicious_package")
199
+ pypi = [a for a in record.affected if a.package.ecosystem == "PyPI"]
200
+ if not pypi:
201
+ return Excluded(id=record.id, reason="no_pypi_package")
202
+
203
+ packages = tuple(
204
+ AffectedPackage(
205
+ name=canonical_package_name(a.package.name),
206
+ raw_name=a.package.name,
207
+ ranges=tuple(
208
+ VersionRange(type=r.type, events=r.events) for r in a.ranges if r.type != "GIT"
209
+ ),
210
+ versions=a.versions,
211
+ )
212
+ for a in pypi
213
+ )
214
+ return Advisory(
215
+ id=record.id,
216
+ source=source_of(record.id),
217
+ aliases=tuple(sorted(set(record.aliases))),
218
+ related=tuple(sorted(set(record.related))),
219
+ published=record.published,
220
+ modified=record.modified,
221
+ summary=record.summary,
222
+ details=record.details,
223
+ packages=packages,
224
+ references=record.references,
225
+ fix_candidates=_fix_candidates(record),
226
+ symbol_hints=_symbol_hints(record),
227
+ )
228
+
229
+
230
+ def iter_dump(zip_path: Path) -> Iterator[tuple[str, dict[str, Any] | None, str | None]]:
231
+ """Yield ``(member name, parsed JSON or None, parse error or None)`` in name order."""
232
+ with zipfile.ZipFile(zip_path) as archive:
233
+ for name in sorted(archive.namelist()):
234
+ if name.endswith("/"):
235
+ continue
236
+ try:
237
+ data = json.loads(archive.read(name))
238
+ except (json.JSONDecodeError, UnicodeDecodeError) as error:
239
+ yield name, None, f"invalid JSON: {error}"
240
+ continue
241
+ if not isinstance(data, dict):
242
+ yield name, None, "top-level JSON value is not an object"
243
+ continue
244
+ yield name, data, None
245
+
246
+
247
+ def normalise_dump(zip_path: Path) -> Iterator[NormaliseResult]:
248
+ """Normalise every record in an OSV dump, in deterministic order."""
249
+ for name, data, error in iter_dump(zip_path):
250
+ if data is None:
251
+ yield Quarantined(source_name=name, reason=error or "unreadable")
252
+ else:
253
+ yield normalise(data, name)
@@ -0,0 +1,189 @@
1
+ """Classify advisory reference URLs and git ranges into candidate fix commits.
2
+
3
+ A candidate is a pointer to code that may contain the fix: a commit, a pull request or a
4
+ tag comparison. Whether it is on the package's own repository, and whether it really is
5
+ the fix, is decided later, when the diff is fetched.
6
+ """
7
+
8
+ import re
9
+ from typing import Literal
10
+ from urllib.parse import parse_qs, urlsplit
11
+
12
+ from pydantic import BaseModel, ConfigDict
13
+
14
+ CandidateKind = Literal[
15
+ "github_commit",
16
+ "github_pull",
17
+ "github_compare",
18
+ "gitlab_commit",
19
+ "gitlab_merge_request",
20
+ "bitbucket_commit",
21
+ "other_commit",
22
+ ]
23
+ CandidateOrigin = Literal["reference", "git_range"]
24
+
25
+ _SHA = r"[0-9a-fA-F]{7,40}"
26
+ _END = r"(?:[/.#?]|$)"
27
+ _GH_REPO = r"^/(?P<owner>[A-Za-z0-9_.-]+)/(?P<repo>[A-Za-z0-9_.-]+)"
28
+ _GH_PR_COMMIT = re.compile(_GH_REPO + rf"/pull/\d+/commits/(?P<sha>{_SHA}){_END}")
29
+ _GH_COMMIT = re.compile(_GH_REPO + rf"/commits?/(?P<sha>{_SHA}){_END}")
30
+ _GH_PULL = re.compile(_GH_REPO + r"/pull/(?P<number>\d+)(?:[/#?]|$)")
31
+ _GH_COMPARE = re.compile(_GH_REPO + r"/compare/(?P<base>[^/?#]+?)\.{2,3}(?P<head>[^/?#]+)")
32
+ _GL_COMMIT = re.compile(rf"^/(?P<repo>.+?)/(?:-/)?commit/(?P<sha>{_SHA}){_END}")
33
+ _GL_MR = re.compile(r"^/(?P<repo>.+?)/(?:-/)?merge_requests/(?P<number>\d+)(?:[/#?]|$)")
34
+ _BB_COMMIT = re.compile(rf"^/(?P<owner>[^/]+)/(?P<repo>[^/]+)/commits?/(?P<sha>{_SHA}){_END}")
35
+ _ANY_COMMIT_PATH = re.compile(rf"/(?:-/)?commits?/(?P<sha>{_SHA}){_END}")
36
+ _GITWEB_HASH = re.compile(r";h=(?P<sha>[0-9a-fA-F]{40})")
37
+ _FULL_SHA = re.compile(r"^[0-9a-fA-F]{40}$")
38
+
39
+
40
+ class FixCandidate(BaseModel):
41
+ """A pointer to code that may contain the fix for an advisory."""
42
+
43
+ model_config = ConfigDict(frozen=True)
44
+
45
+ kind: CandidateKind
46
+ origin: CandidateOrigin
47
+ url: str
48
+ host: str
49
+ repo: str | None = None
50
+ commit: str | None = None
51
+ number: int | None = None
52
+ base: str | None = None
53
+ head: str | None = None
54
+ reference_type: str | None = None
55
+
56
+
57
+ def _clean_repo(name: str) -> str:
58
+ return name.removesuffix(".git").lower()
59
+
60
+
61
+ def _github(path: str) -> dict[str, object] | None:
62
+ for kind, pattern in (
63
+ ("github_commit", _GH_PR_COMMIT),
64
+ ("github_commit", _GH_COMMIT),
65
+ ("github_pull", _GH_PULL),
66
+ ("github_compare", _GH_COMPARE),
67
+ ):
68
+ if match := pattern.match(path):
69
+ fields = match.groupdict()
70
+ result: dict[str, object] = {
71
+ "kind": kind,
72
+ "repo": f"{fields['owner'].lower()}/{_clean_repo(fields['repo'])}",
73
+ }
74
+ if sha := fields.get("sha"):
75
+ result["commit"] = sha.lower()
76
+ if number := fields.get("number"):
77
+ result["number"] = int(number)
78
+ if base := fields.get("base"):
79
+ result["base"], result["head"] = base, fields["head"]
80
+ return result
81
+ return None
82
+
83
+
84
+ def _gitlab(path: str) -> dict[str, object] | None:
85
+ if match := _GL_COMMIT.match(path):
86
+ return {
87
+ "kind": "gitlab_commit",
88
+ "repo": _clean_repo(match["repo"]),
89
+ "commit": match["sha"].lower(),
90
+ }
91
+ if match := _GL_MR.match(path):
92
+ return {
93
+ "kind": "gitlab_merge_request",
94
+ "repo": _clean_repo(match["repo"]),
95
+ "number": int(match["number"]),
96
+ }
97
+ return None
98
+
99
+
100
+ def _bitbucket(path: str) -> dict[str, object] | None:
101
+ if match := _BB_COMMIT.match(path):
102
+ return {
103
+ "kind": "bitbucket_commit",
104
+ "repo": f"{match['owner'].lower()}/{_clean_repo(match['repo'])}",
105
+ "commit": match["sha"].lower(),
106
+ }
107
+ return None
108
+
109
+
110
+ def _self_hosted(url: str, path: str, query: str, host: str) -> dict[str, object] | None:
111
+ """Self-hosted forges: ``/commit/<sha>`` paths, cgit ``?id=<sha>``, gitweb ``;h=<sha>``."""
112
+ sha: str | None = None
113
+ if match := _ANY_COMMIT_PATH.search(path):
114
+ sha = match["sha"]
115
+ else:
116
+ params = parse_qs(query)
117
+ for key in ("id", "h"):
118
+ values = params.get(key, [])
119
+ if values and _FULL_SHA.match(values[0]):
120
+ sha = values[0]
121
+ break
122
+ if sha is None and (match := _GITWEB_HASH.search(url)):
123
+ sha = match["sha"]
124
+ if sha is None or ("commit" not in url and "git" not in host):
125
+ return None
126
+ return {"kind": "other_commit", "commit": sha.lower()}
127
+
128
+
129
+ def classify_url(url: str, reference_type: str | None = None) -> FixCandidate | None:
130
+ """Return a fix candidate if ``url`` points at a commit, pull request or comparison."""
131
+ try:
132
+ parts = urlsplit(url.strip())
133
+ host = (parts.hostname or "").lower()
134
+ except ValueError:
135
+ return None
136
+ if parts.scheme not in ("http", "https") or not host:
137
+ return None
138
+
139
+ if host in ("github.com", "www.github.com"):
140
+ fields = _github(parts.path)
141
+ elif host == "gitlab.com" or host.startswith("gitlab."):
142
+ fields = _gitlab(parts.path)
143
+ elif host == "bitbucket.org":
144
+ fields = _bitbucket(parts.path)
145
+ else:
146
+ fields = _self_hosted(url, parts.path, parts.query, host)
147
+ if fields is None:
148
+ return None
149
+ return FixCandidate.model_validate(
150
+ {
151
+ **fields,
152
+ "origin": "reference",
153
+ "url": url,
154
+ "host": host,
155
+ "reference_type": reference_type,
156
+ }
157
+ )
158
+
159
+
160
+ def candidate_from_git_range(repo_url: str, fixed: str) -> FixCandidate | None:
161
+ """Turn an OSV ``GIT`` range ``fixed`` event into a fix candidate."""
162
+ if not _FULL_SHA.match(fixed):
163
+ return None
164
+ try:
165
+ parts = urlsplit(repo_url.strip())
166
+ host = (parts.hostname or "").lower()
167
+ except ValueError:
168
+ return None
169
+ if not host:
170
+ return None
171
+ path = parts.path.strip("/")
172
+ segments = path.split("/")
173
+ if host in ("github.com", "www.github.com", "bitbucket.org") and len(segments) >= 2:
174
+ kind = "bitbucket_commit" if host == "bitbucket.org" else "github_commit"
175
+ repo: str | None = f"{segments[0].lower()}/{_clean_repo(segments[1])}"
176
+ elif host == "gitlab.com" or host.startswith("gitlab."):
177
+ kind, repo = "gitlab_commit", _clean_repo(path)
178
+ else:
179
+ kind, repo = "other_commit", (_clean_repo(path) or None)
180
+ return FixCandidate.model_validate(
181
+ {
182
+ "kind": kind,
183
+ "origin": "git_range",
184
+ "url": repo_url,
185
+ "host": host,
186
+ "repo": repo,
187
+ "commit": fixed.lower(),
188
+ }
189
+ )
@@ -0,0 +1,146 @@
1
+ """Pinned source snapshots.
2
+
3
+ Every run of the pipeline reads inputs recorded in ``manifests/snapshot.json``: each
4
+ downloaded file with its SHA-256, and each cloned repository with its commit. A file whose
5
+ hash no longer matches the manifest is refused, so results are always traceable to the
6
+ exact inputs that produced them.
7
+ """
8
+
9
+ import hashlib
10
+ import subprocess
11
+ from datetime import UTC, datetime
12
+ from pathlib import Path
13
+
14
+ import httpx
15
+ from pydantic import BaseModel, ConfigDict
16
+
17
+ OSV_PYPI_DUMP_URL = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
18
+ OSV_PYPI_MODIFIED_URL = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/modified_id.csv"
19
+
20
+ _CHUNK = 1 << 20
21
+
22
+
23
+ class SnapshotMismatchError(RuntimeError):
24
+ """A local file does not match the hash pinned in the snapshot manifest."""
25
+
26
+
27
+ class SourceFile(BaseModel):
28
+ model_config = ConfigDict(frozen=True)
29
+
30
+ name: str
31
+ url: str
32
+ path: str
33
+ sha256: str
34
+ size: int
35
+ last_modified: str | None
36
+ retrieved_at: datetime
37
+
38
+
39
+ class SourceRepo(BaseModel):
40
+ model_config = ConfigDict(frozen=True)
41
+
42
+ name: str
43
+ url: str
44
+ path: str
45
+ commit: str
46
+ committed_at: datetime
47
+ note: str = ""
48
+
49
+
50
+ class SnapshotManifest(BaseModel):
51
+ files: tuple[SourceFile, ...] = ()
52
+ repos: tuple[SourceRepo, ...] = ()
53
+
54
+ def file(self, name: str) -> SourceFile:
55
+ for entry in self.files:
56
+ if entry.name == name:
57
+ return entry
58
+ raise KeyError(f"no file named {name!r} in the snapshot manifest")
59
+
60
+ def with_file(self, entry: SourceFile) -> "SnapshotManifest":
61
+ others = tuple(f for f in self.files if f.name != entry.name)
62
+ return self.model_copy(update={"files": tuple(sorted((*others, entry), key=_name))})
63
+
64
+ def with_repo(self, entry: SourceRepo) -> "SnapshotManifest":
65
+ others = tuple(r for r in self.repos if r.name != entry.name)
66
+ return self.model_copy(update={"repos": tuple(sorted((*others, entry), key=_name))})
67
+
68
+
69
+ def _name(entry: SourceFile | SourceRepo) -> str:
70
+ return entry.name
71
+
72
+
73
+ def sha256_file(path: Path) -> str:
74
+ digest = hashlib.sha256()
75
+ with path.open("rb") as handle:
76
+ while chunk := handle.read(_CHUNK):
77
+ digest.update(chunk)
78
+ return digest.hexdigest()
79
+
80
+
81
+ def load_manifest(path: Path) -> SnapshotManifest:
82
+ if not path.exists():
83
+ return SnapshotManifest()
84
+ return SnapshotManifest.model_validate_json(path.read_text(encoding="utf-8"))
85
+
86
+
87
+ def save_manifest(manifest: SnapshotManifest, path: Path) -> None:
88
+ path.parent.mkdir(parents=True, exist_ok=True)
89
+ path.write_text(manifest.model_dump_json(indent=2) + "\n", encoding="utf-8")
90
+
91
+
92
+ def download(client: httpx.Client, name: str, url: str, dest: Path, root: Path) -> SourceFile:
93
+ """Stream ``url`` to ``dest``, hashing as it goes. Replaces ``dest`` only on success."""
94
+ dest.parent.mkdir(parents=True, exist_ok=True)
95
+ partial = dest.with_name(dest.name + ".part")
96
+ digest = hashlib.sha256()
97
+ size = 0
98
+ with client.stream("GET", url) as response:
99
+ response.raise_for_status()
100
+ last_modified = response.headers.get("last-modified")
101
+ with partial.open("wb") as handle:
102
+ for chunk in response.iter_bytes(_CHUNK):
103
+ handle.write(chunk)
104
+ digest.update(chunk)
105
+ size += len(chunk)
106
+ partial.replace(dest)
107
+ return SourceFile(
108
+ name=name,
109
+ url=url,
110
+ path=dest.relative_to(root).as_posix(),
111
+ sha256=digest.hexdigest(),
112
+ size=size,
113
+ last_modified=last_modified,
114
+ retrieved_at=datetime.now(UTC),
115
+ )
116
+
117
+
118
+ def verify(entry: SourceFile, root: Path) -> Path:
119
+ """Return the local path of ``entry`` after checking its hash against the manifest."""
120
+ path = root / entry.path
121
+ if not path.exists():
122
+ raise SnapshotMismatchError(f"{entry.path} is missing; run the snapshot step first")
123
+ actual = sha256_file(path)
124
+ if actual != entry.sha256:
125
+ raise SnapshotMismatchError(
126
+ f"{entry.path} has SHA-256 {actual}, but the manifest pins {entry.sha256}"
127
+ )
128
+ return path
129
+
130
+
131
+ def record_repo(name: str, url: str, path: Path, root: Path, note: str = "") -> SourceRepo:
132
+ """Record the checked-out commit of a local clone."""
133
+ output = subprocess.run( # noqa: S603 - fixed argv, no shell
134
+ ["git", "-C", str(path), "log", "-1", "--format=%H %cI"], # noqa: S607
135
+ capture_output=True,
136
+ text=True,
137
+ check=True,
138
+ ).stdout.split()
139
+ return SourceRepo(
140
+ name=name,
141
+ url=url,
142
+ path=path.relative_to(root).as_posix(),
143
+ commit=output[0],
144
+ committed_at=datetime.fromisoformat(output[1]),
145
+ note=note,
146
+ )
@@ -0,0 +1 @@
1
+ """LLM access for the offline pipeline: one OpenAI-compatible client, cached and rate-limited."""