patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,227 @@
1
+ """Build the symbol database from normalised advisories.
2
+
3
+ This first builder writes *fallback* records: every function, method and class a fix
4
+ changed, at low confidence (proposal §6, step 5). LLM records replace them as extraction
5
+ runs. Low confidence is safe by construction: the scanner lets these symbols raise an alert
6
+ to CALLED, but never lower it to IMPORTED_NOT_CALLED.
7
+
8
+ Fix candidates are used only when they resolve on GitHub, sit on the package's declared
9
+ repository (when one is declared), and change Python source. Tag comparisons are not yet
10
+ resolved; the build report counts them. A fix that touches more than ``MAX_SYMBOLS``
11
+ definitions is reported as ``too_broad`` and gets no record.
12
+ """
13
+
14
+ import hashlib
15
+ import json
16
+ from collections import Counter
17
+ from collections.abc import Callable, Iterable, Sequence
18
+ from datetime import UTC, datetime
19
+
20
+ from pydantic import BaseModel
21
+
22
+ from patchsight.db import FixCommitRef, Record, Symbol
23
+ from patchsight.names import module_for_path
24
+ from patchsight.pipeline.diffs.commits import map_commit
25
+ from patchsight.pipeline.diffs.github import GitHubClient
26
+ from patchsight.pipeline.diffs.paths import classify_path
27
+ from patchsight.pipeline.fix_sample import package_repos
28
+ from patchsight.pipeline.http import HttpError, NotFoundError, RateLimitError
29
+ from patchsight.pipeline.ingest.aliases import AliasGroup
30
+ from patchsight.pipeline.ingest.osv import Advisory
31
+ from patchsight.pipeline.pypi import PyPIClient
32
+
33
+ # Above this many changed definitions a fallback record carries no location signal.
34
+ # Set well beyond the 99th percentile (214 in the first full build).
35
+ MAX_SYMBOLS = 500
36
+
37
+
38
+ class BuildOutcome(BaseModel):
39
+ vulnerability: str
40
+ status: str
41
+ records: int = 0
42
+ symbols: int = 0
43
+ commits: int = 0
44
+
45
+
46
+ class BuildReport(BaseModel):
47
+ db_version: str
48
+ groups: int
49
+ records: int
50
+ symbols: int
51
+ status: dict[str, int]
52
+ outcomes: tuple[BuildOutcome, ...]
53
+
54
+
55
+ def input_hash(group: AliasGroup, by_id: dict[str, Advisory]) -> str:
56
+ """SHA-256 over only the fields that feed extraction (ADR 0001)."""
57
+ payload = []
58
+ for advisory_id in group.advisory_ids:
59
+ a = by_id[advisory_id]
60
+ payload.append(
61
+ {
62
+ "id": a.id,
63
+ "packages": [p.model_dump(mode="json") for p in a.packages],
64
+ "fix_candidates": sorted(
65
+ [c.kind, c.repo, c.commit, c.number, c.base, c.head] for c in a.fix_candidates
66
+ ),
67
+ "summary": a.summary,
68
+ "details": a.details,
69
+ }
70
+ )
71
+ canonical = json.dumps(payload, sort_keys=True, default=str)
72
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
73
+
74
+
75
+ def _python_source(path: str) -> bool:
76
+ return classify_path(path) == "python"
77
+
78
+
79
+ def _fix_commits(
80
+ group: AliasGroup, by_id: dict[str, Advisory], github: GitHubClient
81
+ ) -> tuple[list[tuple[str, str]], int]:
82
+ """(repo, sha) pairs to map, and how many tag comparisons were skipped."""
83
+ found: dict[tuple[str, str], None] = {}
84
+ comparisons = 0
85
+ for advisory_id in group.advisory_ids:
86
+ for c in by_id[advisory_id].fix_candidates:
87
+ if c.kind == "github_commit" and c.repo and c.commit:
88
+ found.setdefault((c.repo, c.commit), None)
89
+ elif c.kind == "github_pull" and c.repo and c.number is not None:
90
+ pull = github.pull(c.repo, c.number)
91
+ if pull.merged and pull.merge_commit_sha:
92
+ found.setdefault((pull.repo, pull.merge_commit_sha), None)
93
+ elif c.kind == "github_compare":
94
+ comparisons += 1
95
+ return list(found), comparisons
96
+
97
+
98
+ def fallback_records(
99
+ group: AliasGroup,
100
+ by_id: dict[str, Advisory],
101
+ github: GitHubClient,
102
+ pypi: PyPIClient,
103
+ now: datetime,
104
+ ) -> tuple[list[Record], BuildOutcome]:
105
+ try:
106
+ packages, repos = package_repos(group, by_id, github, pypi)
107
+ commits, _ = _fix_commits(group, by_id, github)
108
+ except RateLimitError:
109
+ raise
110
+ except (HttpError, KeyError, ValueError):
111
+ return [], BuildOutcome(vulnerability=group.key, status="error")
112
+
113
+ symbols: dict[str, Symbol] = {}
114
+ used: list[FixCommitRef] = []
115
+ module_level_only = False
116
+ for repo, sha in commits:
117
+ try:
118
+ mapping = map_commit(github, repo, sha, include=_python_source)
119
+ except RateLimitError:
120
+ raise
121
+ except (NotFoundError, HttpError, KeyError, ValueError):
122
+ continue
123
+ if repos and mapping.repo not in repos:
124
+ continue
125
+ changed = [(f.mapping.path, c) for f in mapping.files for c in f.mapping.changed]
126
+ if not changed:
127
+ # Python changed, but only at module level (for example registration calls):
128
+ # the paper's "vulnerable function unchanged" case.
129
+ module_level_only = module_level_only or any(
130
+ f.mapping.module_level_hunks for f in mapping.files
131
+ )
132
+ continue
133
+ used.append(FixCommitRef(repo=mapping.repo, sha=mapping.sha))
134
+ for path, definition in changed:
135
+ module = module_for_path(path)
136
+ qualified = f"{module}.{definition.name}" if module else definition.name
137
+ symbols.setdefault(
138
+ qualified,
139
+ Symbol(
140
+ qualified_name=qualified,
141
+ module=module,
142
+ name=definition.name,
143
+ kind=definition.kind,
144
+ path=path,
145
+ ),
146
+ )
147
+
148
+ if not used:
149
+ if module_level_only:
150
+ status = "module_level_only"
151
+ else:
152
+ status = "no_python_fix" if commits else "no_resolvable_commit"
153
+ return [], BuildOutcome(vulnerability=group.key, status=status, commits=len(commits))
154
+ if len(symbols) > MAX_SYMBOLS:
155
+ # A fix that touches thousands of functions (usually a refactor shipped with it)
156
+ # says nothing about where the bug is. Publishing it would make almost any use of
157
+ # the package CALLED; without a record the scanner says UNKNOWN, which is honest.
158
+ return [], BuildOutcome(
159
+ vulnerability=group.key, status="too_broad", symbols=len(symbols), commits=len(used)
160
+ )
161
+ roots = tuple(sorted({s.module.split(".")[0] for s in symbols.values() if s.module}))
162
+ modified = max(by_id[i].modified for i in group.advisory_ids)
163
+ digest = input_hash(group, by_id)
164
+ records = [
165
+ Record(
166
+ vulnerability=group.key,
167
+ package=package,
168
+ confidence="low",
169
+ source="fallback",
170
+ import_roots=roots,
171
+ entry_points=(),
172
+ symbols=tuple(symbols[k] for k in sorted(symbols)),
173
+ fix_commits=tuple(used),
174
+ model=None,
175
+ prompt_version=None,
176
+ input_hash=digest,
177
+ advisory_modified=modified,
178
+ truncated=False,
179
+ extracted_at=now,
180
+ )
181
+ for package in packages
182
+ ]
183
+ return records, BuildOutcome(
184
+ vulnerability=group.key,
185
+ status="built",
186
+ records=len(records),
187
+ symbols=len(symbols),
188
+ commits=len(used),
189
+ )
190
+
191
+
192
+ def build(
193
+ groups: Sequence[AliasGroup],
194
+ by_id: dict[str, Advisory],
195
+ github: GitHubClient,
196
+ pypi: PyPIClient,
197
+ db_version: str,
198
+ progress: Callable[[int, int], None] | None = None,
199
+ ) -> tuple[list[Record], dict[str, str], BuildReport]:
200
+ now = datetime.now(UTC)
201
+ records: list[Record] = []
202
+ aliases: dict[str, str] = {}
203
+ outcomes: list[BuildOutcome] = []
204
+ for number, group in enumerate(groups, 1):
205
+ built, outcome = fallback_records(group, by_id, github, pypi, now)
206
+ if progress is not None and (number % 100 == 0 or number == len(groups)):
207
+ progress(number, len(groups))
208
+ outcomes.append(outcome)
209
+ if built:
210
+ records.extend(built)
211
+ aliases.update(dict.fromkeys(group.identifiers, group.key))
212
+ return (
213
+ records,
214
+ aliases,
215
+ BuildReport(
216
+ db_version=db_version,
217
+ groups=len(groups),
218
+ records=len(records),
219
+ symbols=sum(len(r.symbols) for r in records),
220
+ status=dict(sorted(Counter(o.status for o in outcomes).items())),
221
+ outcomes=tuple(outcomes),
222
+ ),
223
+ )
224
+
225
+
226
+ def jsonl(records: Iterable[Record]) -> str:
227
+ return "".join(r.model_dump_json() + "\n" for r in records)
@@ -0,0 +1 @@
1
+ """Diffs: fetching fix commits, filtering paths, mapping hunks to functions."""
@@ -0,0 +1,109 @@
1
+ """Map a whole fix commit to the definitions it changes: fetch, filter, parse, map."""
2
+
3
+ from collections.abc import Callable
4
+ from typing import Literal
5
+
6
+ from pydantic import BaseModel, ConfigDict
7
+
8
+ from patchsight.pipeline.diffs.github import ChangedFile, GitHubClient
9
+ from patchsight.pipeline.diffs.hunks import Hunk, PatchParseError, diff_hunks, parse_patch
10
+ from patchsight.pipeline.diffs.mapping import (
11
+ FileMapping,
12
+ SourceLanguage,
13
+ language_for,
14
+ map_hunks,
15
+ )
16
+
17
+ DiffSource = Literal["github_patch", "computed"]
18
+ SkipReason = Literal["unsupported_language", "filtered"]
19
+ PathFilter = Callable[[str], bool]
20
+
21
+
22
+ class SkippedFile(BaseModel):
23
+ model_config = ConfigDict(frozen=True)
24
+
25
+ path: str
26
+ reason: SkipReason
27
+
28
+
29
+ class MappedFile(BaseModel):
30
+ model_config = ConfigDict(frozen=True)
31
+
32
+ mapping: FileMapping
33
+ status: str
34
+ previous_path: str | None
35
+ diff_source: DiffSource
36
+ hunks: tuple[Hunk, ...] = ()
37
+
38
+
39
+ class CommitMapping(BaseModel):
40
+ model_config = ConfigDict(frozen=True)
41
+
42
+ repo: str
43
+ sha: str
44
+ parent: str | None
45
+ files: tuple[MappedFile, ...]
46
+ skipped: tuple[SkippedFile, ...]
47
+ truncated: bool
48
+
49
+ def changed_names(self) -> set[str]:
50
+ return {c.name for f in self.files for c in f.mapping.changed}
51
+
52
+
53
+ def _map_file(
54
+ github: GitHubClient,
55
+ repo: str,
56
+ sha: str,
57
+ parent: str | None,
58
+ changed: ChangedFile,
59
+ language: SourceLanguage,
60
+ ) -> MappedFile:
61
+ old_path = changed.previous_filename or changed.filename
62
+ old = None
63
+ if changed.status != "added" and parent is not None:
64
+ old = github.file_content(repo, old_path, parent)
65
+ new = None if changed.status == "removed" else github.file_content(repo, changed.filename, sha)
66
+
67
+ source: DiffSource = "github_patch"
68
+ try:
69
+ if not changed.patch:
70
+ raise PatchParseError("no patch")
71
+ hunks = parse_patch(changed.patch)
72
+ except PatchParseError:
73
+ source = "computed"
74
+ hunks = diff_hunks(
75
+ old.decode("utf-8", "replace") if old is not None else None,
76
+ new.decode("utf-8", "replace") if new is not None else None,
77
+ )
78
+ return MappedFile(
79
+ mapping=map_hunks(changed.filename, language, old, new, hunks),
80
+ status=changed.status,
81
+ previous_path=changed.previous_filename,
82
+ diff_source=source,
83
+ hunks=tuple(hunks),
84
+ )
85
+
86
+
87
+ def map_commit(github: GitHubClient, repo: str, sha: str, include: PathFilter) -> CommitMapping:
88
+ """Map every included Python or Go file of a commit to its changed definitions."""
89
+ change = github.commit(repo, sha)
90
+ parent = change.parents[0] if change.parents else None
91
+ files, skipped = [], []
92
+ for changed in change.files:
93
+ language = language_for(changed.filename)
94
+ if language is None:
95
+ skipped.append(SkippedFile(path=changed.filename, reason="unsupported_language"))
96
+ elif not include(changed.filename):
97
+ skipped.append(SkippedFile(path=changed.filename, reason="filtered"))
98
+ else:
99
+ files.append(
100
+ _map_file(github, change.repo, change.commits[0], parent, changed, language)
101
+ )
102
+ return CommitMapping(
103
+ repo=change.repo,
104
+ sha=change.commits[0],
105
+ parent=parent,
106
+ files=tuple(files),
107
+ skipped=tuple(skipped),
108
+ truncated=change.truncated,
109
+ )
@@ -0,0 +1,177 @@
1
+ """Read fix commits, pull requests and tag comparisons through the GitHub REST API.
2
+
3
+ Read-only, and limited to ``api.github.com``. Repository names come back canonical
4
+ (lower-cased ``owner/name``, after following renames), so they can be compared with the
5
+ package's declared repository.
6
+ """
7
+
8
+ import base64
9
+ import re
10
+ from pathlib import Path
11
+ from typing import Any
12
+ from urllib.parse import quote
13
+
14
+ import httpx
15
+ from pydantic import BaseModel, ConfigDict
16
+ from tenacity.wait import wait_base
17
+
18
+ from patchsight.pipeline.http import CachedJsonClient, NotFoundError
19
+
20
+ GITHUB_API = "https://api.github.com"
21
+ COMMIT_FILES_PER_PAGE = 300 # fixed by the API for single-commit responses
22
+ MAX_COMMIT_PAGES = 10 # the API stops listing files at 3,000
23
+ PULL_FILES_PER_PAGE = 100
24
+ MAX_PULL_PAGES = 30
25
+ COMPARE_FILE_LIMIT = 300
26
+
27
+ _HTML_REPO = re.compile(r"^https://github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/")
28
+
29
+
30
+ class ChangedFile(BaseModel):
31
+ model_config = ConfigDict(frozen=True)
32
+
33
+ filename: str
34
+ status: str
35
+ previous_filename: str | None = None
36
+ additions: int = 0
37
+ deletions: int = 0
38
+ patch: str | None = None
39
+
40
+
41
+ class ChangeSet(BaseModel):
42
+ """The files a commit, merged pull request or comparison changed."""
43
+
44
+ model_config = ConfigDict(frozen=True)
45
+
46
+ repo: str
47
+ commits: tuple[str, ...]
48
+ parents: tuple[str, ...] = ()
49
+ files: tuple[ChangedFile, ...]
50
+ truncated: bool
51
+
52
+
53
+ class PullRequest(BaseModel):
54
+ model_config = ConfigDict(frozen=True)
55
+
56
+ repo: str
57
+ number: int
58
+ merged: bool
59
+ merge_commit_sha: str | None
60
+
61
+
62
+ def _files(raw: list[dict[str, Any]]) -> list[ChangedFile]:
63
+ return [ChangedFile.model_validate(item) for item in raw]
64
+
65
+
66
+ def _repo_from_html(url: str) -> str:
67
+ match = _HTML_REPO.match(url)
68
+ if not match:
69
+ raise ValueError(f"unexpected html_url {url!r}")
70
+ return f"{match['owner']}/{match['repo']}".lower()
71
+
72
+
73
+ class GitHubClient:
74
+ def __init__(
75
+ self,
76
+ cache_dir: Path,
77
+ token: str | None = None,
78
+ transport: httpx.BaseTransport | None = None,
79
+ wait: wait_base | None = None,
80
+ wait_for_rate_limit: float = 0.0,
81
+ ) -> None:
82
+ headers = {
83
+ "Accept": "application/vnd.github+json",
84
+ "X-GitHub-Api-Version": "2022-11-28",
85
+ "User-Agent": "patchsight-pipeline",
86
+ }
87
+ if token:
88
+ headers["Authorization"] = f"Bearer {token}"
89
+ self.http = CachedJsonClient(
90
+ GITHUB_API,
91
+ cache_dir,
92
+ headers=headers,
93
+ transport=transport,
94
+ wait=wait,
95
+ wait_for_rate_limit=wait_for_rate_limit,
96
+ )
97
+
98
+ def close(self) -> None:
99
+ self.http.close()
100
+
101
+ def canonical_repo(self, repo: str) -> str:
102
+ """``owner/name`` as GitHub currently names it, lower-cased. Follows renames."""
103
+ data = self.http.get_json(f"/repos/{repo}")
104
+ return str(data["full_name"]).lower()
105
+
106
+ def commit(self, repo: str, sha: str) -> ChangeSet:
107
+ files: list[ChangedFile] = []
108
+ first: dict[str, Any] = {}
109
+ truncated = False
110
+ for page in range(1, MAX_COMMIT_PAGES + 1):
111
+ data = self.http.get_json(f"/repos/{repo}/commits/{sha}", {"page": page})
112
+ first = first or data
113
+ batch = data.get("files") or []
114
+ files.extend(_files(batch))
115
+ if len(batch) < COMMIT_FILES_PER_PAGE:
116
+ break
117
+ else:
118
+ truncated = True
119
+ return ChangeSet(
120
+ repo=_repo_from_html(first["html_url"]),
121
+ commits=(str(first["sha"]),),
122
+ parents=tuple(str(p["sha"]) for p in first.get("parents") or []),
123
+ files=tuple(files),
124
+ truncated=truncated,
125
+ )
126
+
127
+ def pull(self, repo: str, number: int) -> PullRequest:
128
+ data = self.http.get_json(f"/repos/{repo}/pulls/{number}")
129
+ return PullRequest(
130
+ repo=str(data["base"]["repo"]["full_name"]).lower(),
131
+ number=number,
132
+ merged=bool(data.get("merged")),
133
+ merge_commit_sha=data.get("merge_commit_sha"),
134
+ )
135
+
136
+ def pull_files(self, repo: str, number: int) -> ChangeSet:
137
+ pull = self.pull(repo, number)
138
+ files: list[ChangedFile] = []
139
+ truncated = False
140
+ for page in range(1, MAX_PULL_PAGES + 1):
141
+ batch = self.http.get_json(
142
+ f"/repos/{repo}/pulls/{number}/files",
143
+ {"page": page, "per_page": PULL_FILES_PER_PAGE},
144
+ )
145
+ files.extend(_files(batch))
146
+ if len(batch) < PULL_FILES_PER_PAGE:
147
+ break
148
+ else:
149
+ truncated = True
150
+ commits = (pull.merge_commit_sha,) if pull.merge_commit_sha else ()
151
+ return ChangeSet(repo=pull.repo, commits=commits, files=tuple(files), truncated=truncated)
152
+
153
+ def compare(self, repo: str, base: str, head: str) -> ChangeSet:
154
+ data = self.http.get_json(f"/repos/{repo}/compare/{base}...{head}")
155
+ raw_files = data.get("files") or []
156
+ return ChangeSet(
157
+ repo=_repo_from_html(data["html_url"]),
158
+ commits=tuple(str(c["sha"]) for c in data.get("commits") or []),
159
+ files=tuple(_files(raw_files)),
160
+ truncated=len(raw_files) >= COMPARE_FILE_LIMIT,
161
+ )
162
+
163
+ def file_content(self, repo: str, path: str, ref: str) -> bytes | None:
164
+ """A file's bytes at ``ref``, or None if it does not exist there.
165
+
166
+ The contents API inlines files up to 1 MB; larger ones are read as git blobs.
167
+ """
168
+ try:
169
+ data = self.http.get_json(f"/repos/{repo}/contents/{quote(path)}", {"ref": ref})
170
+ except NotFoundError:
171
+ return None
172
+ if not isinstance(data, dict) or data.get("type") != "file":
173
+ return None
174
+ if data.get("encoding") == "base64" and data.get("content"):
175
+ return base64.b64decode(data["content"])
176
+ blob = self.http.get_json(f"/repos/{repo}/git/blobs/{data['sha']}")
177
+ return base64.b64decode(blob["content"]) if blob.get("content") else b""
@@ -0,0 +1,120 @@
1
+ """Unified-diff hunks: which old and new line numbers a patch changes.
2
+
3
+ GitHub returns each file's patch as bare hunks (no file headers). When it omits the patch,
4
+ because the file is too large or binary, the changed lines are computed from the two full
5
+ file versions instead, and the result records that.
6
+ """
7
+
8
+ import difflib
9
+ import re
10
+
11
+ from pydantic import BaseModel, ConfigDict
12
+
13
+ _HEADER = re.compile(r"^@@ -(?P<old>\d+)(?:,(?P<old_n>\d+))? \+(?P<new>\d+)(?:,(?P<new_n>\d+))? @@")
14
+
15
+
16
+ class Hunk(BaseModel):
17
+ """One hunk. Line numbers are 1-based, in the old (pre-fix) and new (post-fix) file."""
18
+
19
+ model_config = ConfigDict(frozen=True)
20
+
21
+ index: int
22
+ old_start: int
23
+ old_count: int
24
+ new_start: int
25
+ new_count: int
26
+ removed: tuple[int, ...]
27
+ added: tuple[int, ...]
28
+ # The hunk's diff lines, each starting with ' ', '-' or '+'. Used to show the change
29
+ # to the model; never stored in published data.
30
+ lines: tuple[str, ...] = ()
31
+
32
+
33
+ class PatchParseError(ValueError):
34
+ """The patch text is not a sequence of well-formed unified-diff hunks."""
35
+
36
+
37
+ def parse_patch(patch: str) -> list[Hunk]:
38
+ """Parse GitHub's per-file ``patch`` text into hunks."""
39
+ hunks: list[Hunk] = []
40
+ lines = patch.splitlines()
41
+ i = 0
42
+ while i < len(lines):
43
+ match = _HEADER.match(lines[i])
44
+ if not match:
45
+ if lines[i].strip():
46
+ raise PatchParseError(f"expected a hunk header, got {lines[i][:60]!r}")
47
+ i += 1
48
+ continue
49
+ old_start, new_start = int(match["old"]), int(match["new"])
50
+ old_count = int(match["old_n"]) if match["old_n"] is not None else 1
51
+ new_count = int(match["new_n"]) if match["new_n"] is not None else 1
52
+ old_line, new_line = old_start, new_start
53
+ removed: list[int] = []
54
+ added: list[int] = []
55
+ body: list[str] = []
56
+ old_seen = new_seen = 0
57
+ i += 1
58
+ while i < len(lines) and (old_seen < old_count or new_seen < new_count):
59
+ line = lines[i]
60
+ if line.startswith("\\"): # ""
61
+ i += 1
62
+ continue
63
+ marker = line[:1]
64
+ if marker == "-":
65
+ removed.append(old_line)
66
+ old_line += 1
67
+ old_seen += 1
68
+ elif marker == "+":
69
+ added.append(new_line)
70
+ new_line += 1
71
+ new_seen += 1
72
+ elif marker in (" ", ""):
73
+ old_line += 1
74
+ new_line += 1
75
+ old_seen += 1
76
+ new_seen += 1
77
+ else:
78
+ raise PatchParseError(f"unexpected line in hunk: {line[:60]!r}")
79
+ body.append(line if line else " ")
80
+ i += 1
81
+ while i < len(lines) and lines[i].startswith("\\"):
82
+ i += 1
83
+ hunks.append(
84
+ Hunk(
85
+ index=len(hunks),
86
+ old_start=old_start,
87
+ old_count=old_count,
88
+ new_start=new_start,
89
+ new_count=new_count,
90
+ removed=tuple(removed),
91
+ added=tuple(added),
92
+ lines=tuple(body),
93
+ )
94
+ )
95
+ return hunks
96
+
97
+
98
+ def diff_hunks(old_text: str | None, new_text: str | None) -> list[Hunk]:
99
+ """Compute hunks from two full file versions. ``None`` means the file does not exist."""
100
+ old_lines = (old_text or "").splitlines()
101
+ new_lines = (new_text or "").splitlines()
102
+ matcher = difflib.SequenceMatcher(a=old_lines, b=new_lines, autojunk=False)
103
+ hunks: list[Hunk] = []
104
+ for tag, a0, a1, b0, b1 in matcher.get_opcodes():
105
+ if tag == "equal":
106
+ continue
107
+ hunks.append(
108
+ Hunk(
109
+ index=len(hunks),
110
+ old_start=a0 + 1,
111
+ old_count=a1 - a0,
112
+ new_start=b0 + 1,
113
+ new_count=b1 - b0,
114
+ removed=tuple(range(a0 + 1, a1 + 1)),
115
+ added=tuple(range(b0 + 1, b1 + 1)),
116
+ lines=tuple(f"-{x}" for x in old_lines[a0:a1])
117
+ + tuple(f"+{x}" for x in new_lines[b0:b1]),
118
+ )
119
+ )
120
+ return hunks