patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""Build the symbol database from normalised advisories.
|
|
2
|
+
|
|
3
|
+
This first builder writes *fallback* records: every function, method and class a fix
|
|
4
|
+
changed, at low confidence (proposal §6, step 5). LLM records replace them as extraction
|
|
5
|
+
runs. Low confidence is safe by construction: the scanner lets these symbols raise an alert
|
|
6
|
+
to CALLED, but never lower it to IMPORTED_NOT_CALLED.
|
|
7
|
+
|
|
8
|
+
Fix candidates are used only when they resolve on GitHub, sit on the package's declared
|
|
9
|
+
repository (when one is declared), and change Python source. Tag comparisons are not yet
|
|
10
|
+
resolved; the build report counts them. A fix that touches more than ``MAX_SYMBOLS``
|
|
11
|
+
definitions is reported as ``too_broad`` and gets no record.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
from collections import Counter
|
|
17
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel
|
|
21
|
+
|
|
22
|
+
from patchsight.db import FixCommitRef, Record, Symbol
|
|
23
|
+
from patchsight.names import module_for_path
|
|
24
|
+
from patchsight.pipeline.diffs.commits import map_commit
|
|
25
|
+
from patchsight.pipeline.diffs.github import GitHubClient
|
|
26
|
+
from patchsight.pipeline.diffs.paths import classify_path
|
|
27
|
+
from patchsight.pipeline.fix_sample import package_repos
|
|
28
|
+
from patchsight.pipeline.http import HttpError, NotFoundError, RateLimitError
|
|
29
|
+
from patchsight.pipeline.ingest.aliases import AliasGroup
|
|
30
|
+
from patchsight.pipeline.ingest.osv import Advisory
|
|
31
|
+
from patchsight.pipeline.pypi import PyPIClient
|
|
32
|
+
|
|
33
|
+
# Above this many changed definitions a fallback record carries no location signal.
|
|
34
|
+
# Set well beyond the 99th percentile (214 in the first full build).
|
|
35
|
+
MAX_SYMBOLS = 500
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class BuildOutcome(BaseModel):
|
|
39
|
+
vulnerability: str
|
|
40
|
+
status: str
|
|
41
|
+
records: int = 0
|
|
42
|
+
symbols: int = 0
|
|
43
|
+
commits: int = 0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class BuildReport(BaseModel):
|
|
47
|
+
db_version: str
|
|
48
|
+
groups: int
|
|
49
|
+
records: int
|
|
50
|
+
symbols: int
|
|
51
|
+
status: dict[str, int]
|
|
52
|
+
outcomes: tuple[BuildOutcome, ...]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def input_hash(group: AliasGroup, by_id: dict[str, Advisory]) -> str:
|
|
56
|
+
"""SHA-256 over only the fields that feed extraction (ADR 0001)."""
|
|
57
|
+
payload = []
|
|
58
|
+
for advisory_id in group.advisory_ids:
|
|
59
|
+
a = by_id[advisory_id]
|
|
60
|
+
payload.append(
|
|
61
|
+
{
|
|
62
|
+
"id": a.id,
|
|
63
|
+
"packages": [p.model_dump(mode="json") for p in a.packages],
|
|
64
|
+
"fix_candidates": sorted(
|
|
65
|
+
[c.kind, c.repo, c.commit, c.number, c.base, c.head] for c in a.fix_candidates
|
|
66
|
+
),
|
|
67
|
+
"summary": a.summary,
|
|
68
|
+
"details": a.details,
|
|
69
|
+
}
|
|
70
|
+
)
|
|
71
|
+
canonical = json.dumps(payload, sort_keys=True, default=str)
|
|
72
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _python_source(path: str) -> bool:
|
|
76
|
+
return classify_path(path) == "python"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _fix_commits(
|
|
80
|
+
group: AliasGroup, by_id: dict[str, Advisory], github: GitHubClient
|
|
81
|
+
) -> tuple[list[tuple[str, str]], int]:
|
|
82
|
+
"""(repo, sha) pairs to map, and how many tag comparisons were skipped."""
|
|
83
|
+
found: dict[tuple[str, str], None] = {}
|
|
84
|
+
comparisons = 0
|
|
85
|
+
for advisory_id in group.advisory_ids:
|
|
86
|
+
for c in by_id[advisory_id].fix_candidates:
|
|
87
|
+
if c.kind == "github_commit" and c.repo and c.commit:
|
|
88
|
+
found.setdefault((c.repo, c.commit), None)
|
|
89
|
+
elif c.kind == "github_pull" and c.repo and c.number is not None:
|
|
90
|
+
pull = github.pull(c.repo, c.number)
|
|
91
|
+
if pull.merged and pull.merge_commit_sha:
|
|
92
|
+
found.setdefault((pull.repo, pull.merge_commit_sha), None)
|
|
93
|
+
elif c.kind == "github_compare":
|
|
94
|
+
comparisons += 1
|
|
95
|
+
return list(found), comparisons
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def fallback_records(
|
|
99
|
+
group: AliasGroup,
|
|
100
|
+
by_id: dict[str, Advisory],
|
|
101
|
+
github: GitHubClient,
|
|
102
|
+
pypi: PyPIClient,
|
|
103
|
+
now: datetime,
|
|
104
|
+
) -> tuple[list[Record], BuildOutcome]:
|
|
105
|
+
try:
|
|
106
|
+
packages, repos = package_repos(group, by_id, github, pypi)
|
|
107
|
+
commits, _ = _fix_commits(group, by_id, github)
|
|
108
|
+
except RateLimitError:
|
|
109
|
+
raise
|
|
110
|
+
except (HttpError, KeyError, ValueError):
|
|
111
|
+
return [], BuildOutcome(vulnerability=group.key, status="error")
|
|
112
|
+
|
|
113
|
+
symbols: dict[str, Symbol] = {}
|
|
114
|
+
used: list[FixCommitRef] = []
|
|
115
|
+
module_level_only = False
|
|
116
|
+
for repo, sha in commits:
|
|
117
|
+
try:
|
|
118
|
+
mapping = map_commit(github, repo, sha, include=_python_source)
|
|
119
|
+
except RateLimitError:
|
|
120
|
+
raise
|
|
121
|
+
except (NotFoundError, HttpError, KeyError, ValueError):
|
|
122
|
+
continue
|
|
123
|
+
if repos and mapping.repo not in repos:
|
|
124
|
+
continue
|
|
125
|
+
changed = [(f.mapping.path, c) for f in mapping.files for c in f.mapping.changed]
|
|
126
|
+
if not changed:
|
|
127
|
+
# Python changed, but only at module level (for example registration calls):
|
|
128
|
+
# the paper's "vulnerable function unchanged" case.
|
|
129
|
+
module_level_only = module_level_only or any(
|
|
130
|
+
f.mapping.module_level_hunks for f in mapping.files
|
|
131
|
+
)
|
|
132
|
+
continue
|
|
133
|
+
used.append(FixCommitRef(repo=mapping.repo, sha=mapping.sha))
|
|
134
|
+
for path, definition in changed:
|
|
135
|
+
module = module_for_path(path)
|
|
136
|
+
qualified = f"{module}.{definition.name}" if module else definition.name
|
|
137
|
+
symbols.setdefault(
|
|
138
|
+
qualified,
|
|
139
|
+
Symbol(
|
|
140
|
+
qualified_name=qualified,
|
|
141
|
+
module=module,
|
|
142
|
+
name=definition.name,
|
|
143
|
+
kind=definition.kind,
|
|
144
|
+
path=path,
|
|
145
|
+
),
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
if not used:
|
|
149
|
+
if module_level_only:
|
|
150
|
+
status = "module_level_only"
|
|
151
|
+
else:
|
|
152
|
+
status = "no_python_fix" if commits else "no_resolvable_commit"
|
|
153
|
+
return [], BuildOutcome(vulnerability=group.key, status=status, commits=len(commits))
|
|
154
|
+
if len(symbols) > MAX_SYMBOLS:
|
|
155
|
+
# A fix that touches thousands of functions (usually a refactor shipped with it)
|
|
156
|
+
# says nothing about where the bug is. Publishing it would make almost any use of
|
|
157
|
+
# the package CALLED; without a record the scanner says UNKNOWN, which is honest.
|
|
158
|
+
return [], BuildOutcome(
|
|
159
|
+
vulnerability=group.key, status="too_broad", symbols=len(symbols), commits=len(used)
|
|
160
|
+
)
|
|
161
|
+
roots = tuple(sorted({s.module.split(".")[0] for s in symbols.values() if s.module}))
|
|
162
|
+
modified = max(by_id[i].modified for i in group.advisory_ids)
|
|
163
|
+
digest = input_hash(group, by_id)
|
|
164
|
+
records = [
|
|
165
|
+
Record(
|
|
166
|
+
vulnerability=group.key,
|
|
167
|
+
package=package,
|
|
168
|
+
confidence="low",
|
|
169
|
+
source="fallback",
|
|
170
|
+
import_roots=roots,
|
|
171
|
+
entry_points=(),
|
|
172
|
+
symbols=tuple(symbols[k] for k in sorted(symbols)),
|
|
173
|
+
fix_commits=tuple(used),
|
|
174
|
+
model=None,
|
|
175
|
+
prompt_version=None,
|
|
176
|
+
input_hash=digest,
|
|
177
|
+
advisory_modified=modified,
|
|
178
|
+
truncated=False,
|
|
179
|
+
extracted_at=now,
|
|
180
|
+
)
|
|
181
|
+
for package in packages
|
|
182
|
+
]
|
|
183
|
+
return records, BuildOutcome(
|
|
184
|
+
vulnerability=group.key,
|
|
185
|
+
status="built",
|
|
186
|
+
records=len(records),
|
|
187
|
+
symbols=len(symbols),
|
|
188
|
+
commits=len(used),
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def build(
|
|
193
|
+
groups: Sequence[AliasGroup],
|
|
194
|
+
by_id: dict[str, Advisory],
|
|
195
|
+
github: GitHubClient,
|
|
196
|
+
pypi: PyPIClient,
|
|
197
|
+
db_version: str,
|
|
198
|
+
progress: Callable[[int, int], None] | None = None,
|
|
199
|
+
) -> tuple[list[Record], dict[str, str], BuildReport]:
|
|
200
|
+
now = datetime.now(UTC)
|
|
201
|
+
records: list[Record] = []
|
|
202
|
+
aliases: dict[str, str] = {}
|
|
203
|
+
outcomes: list[BuildOutcome] = []
|
|
204
|
+
for number, group in enumerate(groups, 1):
|
|
205
|
+
built, outcome = fallback_records(group, by_id, github, pypi, now)
|
|
206
|
+
if progress is not None and (number % 100 == 0 or number == len(groups)):
|
|
207
|
+
progress(number, len(groups))
|
|
208
|
+
outcomes.append(outcome)
|
|
209
|
+
if built:
|
|
210
|
+
records.extend(built)
|
|
211
|
+
aliases.update(dict.fromkeys(group.identifiers, group.key))
|
|
212
|
+
return (
|
|
213
|
+
records,
|
|
214
|
+
aliases,
|
|
215
|
+
BuildReport(
|
|
216
|
+
db_version=db_version,
|
|
217
|
+
groups=len(groups),
|
|
218
|
+
records=len(records),
|
|
219
|
+
symbols=sum(len(r.symbols) for r in records),
|
|
220
|
+
status=dict(sorted(Counter(o.status for o in outcomes).items())),
|
|
221
|
+
outcomes=tuple(outcomes),
|
|
222
|
+
),
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def jsonl(records: Iterable[Record]) -> str:
|
|
227
|
+
return "".join(r.model_dump_json() + "\n" for r in records)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Diffs: fetching fix commits, filtering paths, mapping hunks to functions."""
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Map a whole fix commit to the definitions it changes: fetch, filter, parse, map."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict
|
|
7
|
+
|
|
8
|
+
from patchsight.pipeline.diffs.github import ChangedFile, GitHubClient
|
|
9
|
+
from patchsight.pipeline.diffs.hunks import Hunk, PatchParseError, diff_hunks, parse_patch
|
|
10
|
+
from patchsight.pipeline.diffs.mapping import (
|
|
11
|
+
FileMapping,
|
|
12
|
+
SourceLanguage,
|
|
13
|
+
language_for,
|
|
14
|
+
map_hunks,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
DiffSource = Literal["github_patch", "computed"]
|
|
18
|
+
SkipReason = Literal["unsupported_language", "filtered"]
|
|
19
|
+
PathFilter = Callable[[str], bool]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SkippedFile(BaseModel):
|
|
23
|
+
model_config = ConfigDict(frozen=True)
|
|
24
|
+
|
|
25
|
+
path: str
|
|
26
|
+
reason: SkipReason
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class MappedFile(BaseModel):
|
|
30
|
+
model_config = ConfigDict(frozen=True)
|
|
31
|
+
|
|
32
|
+
mapping: FileMapping
|
|
33
|
+
status: str
|
|
34
|
+
previous_path: str | None
|
|
35
|
+
diff_source: DiffSource
|
|
36
|
+
hunks: tuple[Hunk, ...] = ()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class CommitMapping(BaseModel):
|
|
40
|
+
model_config = ConfigDict(frozen=True)
|
|
41
|
+
|
|
42
|
+
repo: str
|
|
43
|
+
sha: str
|
|
44
|
+
parent: str | None
|
|
45
|
+
files: tuple[MappedFile, ...]
|
|
46
|
+
skipped: tuple[SkippedFile, ...]
|
|
47
|
+
truncated: bool
|
|
48
|
+
|
|
49
|
+
def changed_names(self) -> set[str]:
|
|
50
|
+
return {c.name for f in self.files for c in f.mapping.changed}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _map_file(
|
|
54
|
+
github: GitHubClient,
|
|
55
|
+
repo: str,
|
|
56
|
+
sha: str,
|
|
57
|
+
parent: str | None,
|
|
58
|
+
changed: ChangedFile,
|
|
59
|
+
language: SourceLanguage,
|
|
60
|
+
) -> MappedFile:
|
|
61
|
+
old_path = changed.previous_filename or changed.filename
|
|
62
|
+
old = None
|
|
63
|
+
if changed.status != "added" and parent is not None:
|
|
64
|
+
old = github.file_content(repo, old_path, parent)
|
|
65
|
+
new = None if changed.status == "removed" else github.file_content(repo, changed.filename, sha)
|
|
66
|
+
|
|
67
|
+
source: DiffSource = "github_patch"
|
|
68
|
+
try:
|
|
69
|
+
if not changed.patch:
|
|
70
|
+
raise PatchParseError("no patch")
|
|
71
|
+
hunks = parse_patch(changed.patch)
|
|
72
|
+
except PatchParseError:
|
|
73
|
+
source = "computed"
|
|
74
|
+
hunks = diff_hunks(
|
|
75
|
+
old.decode("utf-8", "replace") if old is not None else None,
|
|
76
|
+
new.decode("utf-8", "replace") if new is not None else None,
|
|
77
|
+
)
|
|
78
|
+
return MappedFile(
|
|
79
|
+
mapping=map_hunks(changed.filename, language, old, new, hunks),
|
|
80
|
+
status=changed.status,
|
|
81
|
+
previous_path=changed.previous_filename,
|
|
82
|
+
diff_source=source,
|
|
83
|
+
hunks=tuple(hunks),
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def map_commit(github: GitHubClient, repo: str, sha: str, include: PathFilter) -> CommitMapping:
|
|
88
|
+
"""Map every included Python or Go file of a commit to its changed definitions."""
|
|
89
|
+
change = github.commit(repo, sha)
|
|
90
|
+
parent = change.parents[0] if change.parents else None
|
|
91
|
+
files, skipped = [], []
|
|
92
|
+
for changed in change.files:
|
|
93
|
+
language = language_for(changed.filename)
|
|
94
|
+
if language is None:
|
|
95
|
+
skipped.append(SkippedFile(path=changed.filename, reason="unsupported_language"))
|
|
96
|
+
elif not include(changed.filename):
|
|
97
|
+
skipped.append(SkippedFile(path=changed.filename, reason="filtered"))
|
|
98
|
+
else:
|
|
99
|
+
files.append(
|
|
100
|
+
_map_file(github, change.repo, change.commits[0], parent, changed, language)
|
|
101
|
+
)
|
|
102
|
+
return CommitMapping(
|
|
103
|
+
repo=change.repo,
|
|
104
|
+
sha=change.commits[0],
|
|
105
|
+
parent=parent,
|
|
106
|
+
files=tuple(files),
|
|
107
|
+
skipped=tuple(skipped),
|
|
108
|
+
truncated=change.truncated,
|
|
109
|
+
)
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Read fix commits, pull requests and tag comparisons through the GitHub REST API.
|
|
2
|
+
|
|
3
|
+
Read-only, and limited to ``api.github.com``. Repository names come back canonical
|
|
4
|
+
(lower-cased ``owner/name``, after following renames), so they can be compared with the
|
|
5
|
+
package's declared repository.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import base64
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
from urllib.parse import quote
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
from pydantic import BaseModel, ConfigDict
|
|
16
|
+
from tenacity.wait import wait_base
|
|
17
|
+
|
|
18
|
+
from patchsight.pipeline.http import CachedJsonClient, NotFoundError
|
|
19
|
+
|
|
20
|
+
GITHUB_API = "https://api.github.com"
|
|
21
|
+
COMMIT_FILES_PER_PAGE = 300 # fixed by the API for single-commit responses
|
|
22
|
+
MAX_COMMIT_PAGES = 10 # the API stops listing files at 3,000
|
|
23
|
+
PULL_FILES_PER_PAGE = 100
|
|
24
|
+
MAX_PULL_PAGES = 30
|
|
25
|
+
COMPARE_FILE_LIMIT = 300
|
|
26
|
+
|
|
27
|
+
_HTML_REPO = re.compile(r"^https://github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ChangedFile(BaseModel):
|
|
31
|
+
model_config = ConfigDict(frozen=True)
|
|
32
|
+
|
|
33
|
+
filename: str
|
|
34
|
+
status: str
|
|
35
|
+
previous_filename: str | None = None
|
|
36
|
+
additions: int = 0
|
|
37
|
+
deletions: int = 0
|
|
38
|
+
patch: str | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ChangeSet(BaseModel):
|
|
42
|
+
"""The files a commit, merged pull request or comparison changed."""
|
|
43
|
+
|
|
44
|
+
model_config = ConfigDict(frozen=True)
|
|
45
|
+
|
|
46
|
+
repo: str
|
|
47
|
+
commits: tuple[str, ...]
|
|
48
|
+
parents: tuple[str, ...] = ()
|
|
49
|
+
files: tuple[ChangedFile, ...]
|
|
50
|
+
truncated: bool
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class PullRequest(BaseModel):
|
|
54
|
+
model_config = ConfigDict(frozen=True)
|
|
55
|
+
|
|
56
|
+
repo: str
|
|
57
|
+
number: int
|
|
58
|
+
merged: bool
|
|
59
|
+
merge_commit_sha: str | None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _files(raw: list[dict[str, Any]]) -> list[ChangedFile]:
|
|
63
|
+
return [ChangedFile.model_validate(item) for item in raw]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _repo_from_html(url: str) -> str:
|
|
67
|
+
match = _HTML_REPO.match(url)
|
|
68
|
+
if not match:
|
|
69
|
+
raise ValueError(f"unexpected html_url {url!r}")
|
|
70
|
+
return f"{match['owner']}/{match['repo']}".lower()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class GitHubClient:
|
|
74
|
+
def __init__(
|
|
75
|
+
self,
|
|
76
|
+
cache_dir: Path,
|
|
77
|
+
token: str | None = None,
|
|
78
|
+
transport: httpx.BaseTransport | None = None,
|
|
79
|
+
wait: wait_base | None = None,
|
|
80
|
+
wait_for_rate_limit: float = 0.0,
|
|
81
|
+
) -> None:
|
|
82
|
+
headers = {
|
|
83
|
+
"Accept": "application/vnd.github+json",
|
|
84
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
85
|
+
"User-Agent": "patchsight-pipeline",
|
|
86
|
+
}
|
|
87
|
+
if token:
|
|
88
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
89
|
+
self.http = CachedJsonClient(
|
|
90
|
+
GITHUB_API,
|
|
91
|
+
cache_dir,
|
|
92
|
+
headers=headers,
|
|
93
|
+
transport=transport,
|
|
94
|
+
wait=wait,
|
|
95
|
+
wait_for_rate_limit=wait_for_rate_limit,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def close(self) -> None:
|
|
99
|
+
self.http.close()
|
|
100
|
+
|
|
101
|
+
def canonical_repo(self, repo: str) -> str:
|
|
102
|
+
"""``owner/name`` as GitHub currently names it, lower-cased. Follows renames."""
|
|
103
|
+
data = self.http.get_json(f"/repos/{repo}")
|
|
104
|
+
return str(data["full_name"]).lower()
|
|
105
|
+
|
|
106
|
+
def commit(self, repo: str, sha: str) -> ChangeSet:
|
|
107
|
+
files: list[ChangedFile] = []
|
|
108
|
+
first: dict[str, Any] = {}
|
|
109
|
+
truncated = False
|
|
110
|
+
for page in range(1, MAX_COMMIT_PAGES + 1):
|
|
111
|
+
data = self.http.get_json(f"/repos/{repo}/commits/{sha}", {"page": page})
|
|
112
|
+
first = first or data
|
|
113
|
+
batch = data.get("files") or []
|
|
114
|
+
files.extend(_files(batch))
|
|
115
|
+
if len(batch) < COMMIT_FILES_PER_PAGE:
|
|
116
|
+
break
|
|
117
|
+
else:
|
|
118
|
+
truncated = True
|
|
119
|
+
return ChangeSet(
|
|
120
|
+
repo=_repo_from_html(first["html_url"]),
|
|
121
|
+
commits=(str(first["sha"]),),
|
|
122
|
+
parents=tuple(str(p["sha"]) for p in first.get("parents") or []),
|
|
123
|
+
files=tuple(files),
|
|
124
|
+
truncated=truncated,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
def pull(self, repo: str, number: int) -> PullRequest:
|
|
128
|
+
data = self.http.get_json(f"/repos/{repo}/pulls/{number}")
|
|
129
|
+
return PullRequest(
|
|
130
|
+
repo=str(data["base"]["repo"]["full_name"]).lower(),
|
|
131
|
+
number=number,
|
|
132
|
+
merged=bool(data.get("merged")),
|
|
133
|
+
merge_commit_sha=data.get("merge_commit_sha"),
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
def pull_files(self, repo: str, number: int) -> ChangeSet:
|
|
137
|
+
pull = self.pull(repo, number)
|
|
138
|
+
files: list[ChangedFile] = []
|
|
139
|
+
truncated = False
|
|
140
|
+
for page in range(1, MAX_PULL_PAGES + 1):
|
|
141
|
+
batch = self.http.get_json(
|
|
142
|
+
f"/repos/{repo}/pulls/{number}/files",
|
|
143
|
+
{"page": page, "per_page": PULL_FILES_PER_PAGE},
|
|
144
|
+
)
|
|
145
|
+
files.extend(_files(batch))
|
|
146
|
+
if len(batch) < PULL_FILES_PER_PAGE:
|
|
147
|
+
break
|
|
148
|
+
else:
|
|
149
|
+
truncated = True
|
|
150
|
+
commits = (pull.merge_commit_sha,) if pull.merge_commit_sha else ()
|
|
151
|
+
return ChangeSet(repo=pull.repo, commits=commits, files=tuple(files), truncated=truncated)
|
|
152
|
+
|
|
153
|
+
def compare(self, repo: str, base: str, head: str) -> ChangeSet:
|
|
154
|
+
data = self.http.get_json(f"/repos/{repo}/compare/{base}...{head}")
|
|
155
|
+
raw_files = data.get("files") or []
|
|
156
|
+
return ChangeSet(
|
|
157
|
+
repo=_repo_from_html(data["html_url"]),
|
|
158
|
+
commits=tuple(str(c["sha"]) for c in data.get("commits") or []),
|
|
159
|
+
files=tuple(_files(raw_files)),
|
|
160
|
+
truncated=len(raw_files) >= COMPARE_FILE_LIMIT,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def file_content(self, repo: str, path: str, ref: str) -> bytes | None:
|
|
164
|
+
"""A file's bytes at ``ref``, or None if it does not exist there.
|
|
165
|
+
|
|
166
|
+
The contents API inlines files up to 1 MB; larger ones are read as git blobs.
|
|
167
|
+
"""
|
|
168
|
+
try:
|
|
169
|
+
data = self.http.get_json(f"/repos/{repo}/contents/{quote(path)}", {"ref": ref})
|
|
170
|
+
except NotFoundError:
|
|
171
|
+
return None
|
|
172
|
+
if not isinstance(data, dict) or data.get("type") != "file":
|
|
173
|
+
return None
|
|
174
|
+
if data.get("encoding") == "base64" and data.get("content"):
|
|
175
|
+
return base64.b64decode(data["content"])
|
|
176
|
+
blob = self.http.get_json(f"/repos/{repo}/git/blobs/{data['sha']}")
|
|
177
|
+
return base64.b64decode(blob["content"]) if blob.get("content") else b""
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Unified-diff hunks: which old and new line numbers a patch changes.
|
|
2
|
+
|
|
3
|
+
GitHub returns each file's patch as bare hunks (no file headers). When it omits the patch,
|
|
4
|
+
because the file is too large or binary, the changed lines are computed from the two full
|
|
5
|
+
file versions instead, and the result records that.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import difflib
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel, ConfigDict
|
|
12
|
+
|
|
13
|
+
_HEADER = re.compile(r"^@@ -(?P<old>\d+)(?:,(?P<old_n>\d+))? \+(?P<new>\d+)(?:,(?P<new_n>\d+))? @@")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Hunk(BaseModel):
|
|
17
|
+
"""One hunk. Line numbers are 1-based, in the old (pre-fix) and new (post-fix) file."""
|
|
18
|
+
|
|
19
|
+
model_config = ConfigDict(frozen=True)
|
|
20
|
+
|
|
21
|
+
index: int
|
|
22
|
+
old_start: int
|
|
23
|
+
old_count: int
|
|
24
|
+
new_start: int
|
|
25
|
+
new_count: int
|
|
26
|
+
removed: tuple[int, ...]
|
|
27
|
+
added: tuple[int, ...]
|
|
28
|
+
# The hunk's diff lines, each starting with ' ', '-' or '+'. Used to show the change
|
|
29
|
+
# to the model; never stored in published data.
|
|
30
|
+
lines: tuple[str, ...] = ()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class PatchParseError(ValueError):
|
|
34
|
+
"""The patch text is not a sequence of well-formed unified-diff hunks."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def parse_patch(patch: str) -> list[Hunk]:
|
|
38
|
+
"""Parse GitHub's per-file ``patch`` text into hunks."""
|
|
39
|
+
hunks: list[Hunk] = []
|
|
40
|
+
lines = patch.splitlines()
|
|
41
|
+
i = 0
|
|
42
|
+
while i < len(lines):
|
|
43
|
+
match = _HEADER.match(lines[i])
|
|
44
|
+
if not match:
|
|
45
|
+
if lines[i].strip():
|
|
46
|
+
raise PatchParseError(f"expected a hunk header, got {lines[i][:60]!r}")
|
|
47
|
+
i += 1
|
|
48
|
+
continue
|
|
49
|
+
old_start, new_start = int(match["old"]), int(match["new"])
|
|
50
|
+
old_count = int(match["old_n"]) if match["old_n"] is not None else 1
|
|
51
|
+
new_count = int(match["new_n"]) if match["new_n"] is not None else 1
|
|
52
|
+
old_line, new_line = old_start, new_start
|
|
53
|
+
removed: list[int] = []
|
|
54
|
+
added: list[int] = []
|
|
55
|
+
body: list[str] = []
|
|
56
|
+
old_seen = new_seen = 0
|
|
57
|
+
i += 1
|
|
58
|
+
while i < len(lines) and (old_seen < old_count or new_seen < new_count):
|
|
59
|
+
line = lines[i]
|
|
60
|
+
if line.startswith("\\"): # ""
|
|
61
|
+
i += 1
|
|
62
|
+
continue
|
|
63
|
+
marker = line[:1]
|
|
64
|
+
if marker == "-":
|
|
65
|
+
removed.append(old_line)
|
|
66
|
+
old_line += 1
|
|
67
|
+
old_seen += 1
|
|
68
|
+
elif marker == "+":
|
|
69
|
+
added.append(new_line)
|
|
70
|
+
new_line += 1
|
|
71
|
+
new_seen += 1
|
|
72
|
+
elif marker in (" ", ""):
|
|
73
|
+
old_line += 1
|
|
74
|
+
new_line += 1
|
|
75
|
+
old_seen += 1
|
|
76
|
+
new_seen += 1
|
|
77
|
+
else:
|
|
78
|
+
raise PatchParseError(f"unexpected line in hunk: {line[:60]!r}")
|
|
79
|
+
body.append(line if line else " ")
|
|
80
|
+
i += 1
|
|
81
|
+
while i < len(lines) and lines[i].startswith("\\"):
|
|
82
|
+
i += 1
|
|
83
|
+
hunks.append(
|
|
84
|
+
Hunk(
|
|
85
|
+
index=len(hunks),
|
|
86
|
+
old_start=old_start,
|
|
87
|
+
old_count=old_count,
|
|
88
|
+
new_start=new_start,
|
|
89
|
+
new_count=new_count,
|
|
90
|
+
removed=tuple(removed),
|
|
91
|
+
added=tuple(added),
|
|
92
|
+
lines=tuple(body),
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
return hunks
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def diff_hunks(old_text: str | None, new_text: str | None) -> list[Hunk]:
|
|
99
|
+
"""Compute hunks from two full file versions. ``None`` means the file does not exist."""
|
|
100
|
+
old_lines = (old_text or "").splitlines()
|
|
101
|
+
new_lines = (new_text or "").splitlines()
|
|
102
|
+
matcher = difflib.SequenceMatcher(a=old_lines, b=new_lines, autojunk=False)
|
|
103
|
+
hunks: list[Hunk] = []
|
|
104
|
+
for tag, a0, a1, b0, b1 in matcher.get_opcodes():
|
|
105
|
+
if tag == "equal":
|
|
106
|
+
continue
|
|
107
|
+
hunks.append(
|
|
108
|
+
Hunk(
|
|
109
|
+
index=len(hunks),
|
|
110
|
+
old_start=a0 + 1,
|
|
111
|
+
old_count=a1 - a0,
|
|
112
|
+
new_start=b0 + 1,
|
|
113
|
+
new_count=b1 - b0,
|
|
114
|
+
removed=tuple(range(a0 + 1, a1 + 1)),
|
|
115
|
+
added=tuple(range(b0 + 1, b1 + 1)),
|
|
116
|
+
lines=tuple(f"-{x}" for x in old_lines[a0:a1])
|
|
117
|
+
+ tuple(f"+{x}" for x in new_lines[b0:b1]),
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
return hunks
|