patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1 @@
1
+ """Published benchmarks used to evaluate extraction."""
@@ -0,0 +1,108 @@
1
+ """Benchmark bundles: what a stranger needs to recompute the benchmark scores.
2
+
3
+ The fix diffs cannot be redistributed, because they are other projects' code. The scores do
4
+ not need them, though: they need only which functions each fix changed. A bundle records
5
+ that metadata (commit SHAs, file paths, function names and kinds) together with each report's
6
+ ground-truth symbols, so ``reproduce`` recomputes the tables offline, with no API key.
7
+
8
+ Contents and licences: candidate metadata derived by PatchSight from public fix commits;
9
+ ground-truth symbols from GoVulnDB as released by Dunlap et al. (GoVulnDB is CC BY 4.0; the
10
+ Dunlap et al. data is Apache-2.0).
11
+ """
12
+
13
+ import json
14
+ from collections.abc import Sequence
15
+ from pathlib import Path
16
+
17
+ from pydantic import BaseModel, ConfigDict
18
+
19
+ from patchsight.eval.metrics import SetScores, score
20
+ from patchsight.pipeline.benchmarks import dunlap
21
+ from patchsight.pipeline.diffs.commits import map_commit
22
+ from patchsight.pipeline.diffs.github import GitHubClient
23
+
24
+ BUNDLE_VERSION = 1
25
+
26
+
27
+ class BundleCandidate(BaseModel):
28
+ model_config = ConfigDict(frozen=True)
29
+
30
+ commit: str
31
+ path: str
32
+ name: str
33
+ kind: str
34
+
35
+
36
+ class BundleReport(BaseModel):
37
+ model_config = ConfigDict(frozen=True)
38
+
39
+ id: str
40
+ symbols: tuple[str, ...]
41
+ commits: tuple[str, ...]
42
+ candidates: tuple[BundleCandidate, ...]
43
+
44
+ def changed_functions(self) -> set[str]:
45
+ """ChangedFuncs' prediction: every changed function or method, by name."""
46
+ return {c.name for c in self.candidates if c.kind in ("function", "method")}
47
+
48
+
49
+ class Bundle(BaseModel):
50
+ bundle_version: int
51
+ benchmark: str
52
+ source: dict[str, str]
53
+ path_rule: str
54
+ reports: tuple[BundleReport, ...]
55
+
56
+
57
+ def export(
58
+ github: GitHubClient, reports: Sequence[dunlap.DunlapReport], source: dict[str, str]
59
+ ) -> Bundle:
60
+ """Map every report's fix commits (from the response cache when present)."""
61
+ exported = []
62
+ for report in reports:
63
+ candidates: list[BundleCandidate] = []
64
+ for commit in report.commits:
65
+ mapping = map_commit(
66
+ github, commit.github_repo, commit.sha, include=dunlap.paper_path_filter
67
+ )
68
+ candidates.extend(
69
+ BundleCandidate(commit=commit.sha, path=f.mapping.path, name=c.name, kind=c.kind)
70
+ for f in mapping.files
71
+ for c in f.mapping.changed
72
+ )
73
+ exported.append(
74
+ BundleReport(
75
+ id=report.id,
76
+ symbols=tuple(sorted(report.symbols)),
77
+ commits=tuple(c.sha for c in report.commits),
78
+ candidates=tuple(sorted(candidates, key=lambda c: (c.commit, c.path, c.name))),
79
+ )
80
+ )
81
+ return Bundle(
82
+ bundle_version=BUNDLE_VERSION,
83
+ benchmark="dunlap-dimva24",
84
+ source=source,
85
+ path_rule='Go files whose path does not contain "test" (the paper\'s rule)',
86
+ reports=tuple(exported),
87
+ )
88
+
89
+
90
+ def save(bundle: Bundle, path: Path) -> None:
91
+ path.parent.mkdir(parents=True, exist_ok=True)
92
+ path.write_text(json.dumps(bundle.model_dump(mode="json"), indent=1) + "\n", encoding="utf-8")
93
+
94
+
95
+ def load(path: Path) -> Bundle:
96
+ bundle = Bundle.model_validate_json(path.read_text(encoding="utf-8"))
97
+ if bundle.bundle_version != BUNDLE_VERSION:
98
+ raise ValueError(f"{path} is bundle version {bundle.bundle_version}, not {BUNDLE_VERSION}")
99
+ return bundle
100
+
101
+
102
+ def changedfuncs_scores(
103
+ bundle: Bundle, ids: frozenset[str] | None = None, bootstrap: int = 0, seed: int = 0
104
+ ) -> SetScores:
105
+ chosen = [r for r in bundle.reports if ids is None or r.id in ids]
106
+ predicted = {r.id: r.changed_functions() for r in chosen}
107
+ truth = {r.id: set(r.symbols) for r in chosen}
108
+ return score(predicted, truth, bootstrap, seed)
@@ -0,0 +1,275 @@
1
+ """Reproduce the ChangedFuncs baseline on the Dunlap benchmark (proposal §15, Phase 1 step 3).
2
+
3
+ ChangedFuncs marks every function a fix commit changes as vulnerable. Running it before
4
+ any LLM proves the harness: diff fetching, hunk mapping and scoring. It is scored three ways:
5
+
6
+ 1. the paper's published numbers, as reported;
7
+ 2. the authors' released candidate lists, scored by PatchSight's metric code;
8
+ 3. PatchSight's own pipeline: the same 298 commits fetched from GitHub and mapped with
9
+ PatchSight's tree-sitter code, using the paper's path rule.
10
+
11
+ Agreement between 2 and 3 is the direct check on the diff and mapping code.
12
+ """
13
+
14
+ from collections import Counter
15
+ from collections.abc import Sequence
16
+ from datetime import date
17
+
18
+ from pydantic import BaseModel, ConfigDict
19
+
20
+ from patchsight.eval.metrics import Interval, SetScores, jaccard, score
21
+ from patchsight.pipeline.benchmarks import dunlap
22
+ from patchsight.pipeline.diffs.commits import CommitMapping, map_commit
23
+ from patchsight.pipeline.diffs.github import GitHubClient
24
+ from patchsight.pipeline.http import HttpError, NotFoundError, RateLimitError
25
+
26
+ BOOTSTRAP = 2000
27
+ SEED = 20261114
28
+
29
+
30
+ class CommitOutcome(BaseModel):
31
+ model_config = ConfigDict(frozen=True)
32
+
33
+ report: str
34
+ repo: str
35
+ sha: str
36
+ status: str
37
+ files_mapped: int = 0
38
+ computed_diffs: int = 0
39
+ parse_errors: int = 0
40
+ truncated: bool = False
41
+
42
+
43
+ class ReportComparison(BaseModel):
44
+ model_config = ConfigDict(frozen=True)
45
+
46
+ id: str
47
+ symbols: tuple[str, ...]
48
+ ours: tuple[str, ...]
49
+ released: tuple[str, ...]
50
+ jaccard: float
51
+
52
+
53
+ class BaselineReport(BaseModel):
54
+ as_of: date
55
+ reports: int
56
+ commits: int
57
+ unique_commits: int
58
+ truth_symbols: int
59
+ paper: dict[str, float | int]
60
+ paper_unit_released: dict[str, float | int]
61
+ released_unfiltered: SetScores
62
+ released_filtered: SetScores
63
+ ours: SetScores
64
+ commit_status: dict[str, int]
65
+ computed_diffs: int
66
+ parse_errors: int
67
+ truncated_commits: int
68
+ candidates_ours: int
69
+ candidates_released: int
70
+ identical_reports: int
71
+ mean_jaccard: float
72
+ released_found: int
73
+ superset_reports: int
74
+ extra_candidates: int
75
+ extra_in_top5_reports: int
76
+ comparisons: tuple[ReportComparison, ...]
77
+ commit_outcomes: tuple[CommitOutcome, ...]
78
+
79
+
80
+ def _map(
81
+ github: GitHubClient, report: str, commit: dunlap.FixCommit
82
+ ) -> tuple[CommitOutcome, CommitMapping | None]:
83
+ try:
84
+ mapping = map_commit(
85
+ github, commit.github_repo, commit.sha, include=dunlap.paper_path_filter
86
+ )
87
+ except RateLimitError:
88
+ raise
89
+ except NotFoundError:
90
+ return CommitOutcome(
91
+ report=report, repo=commit.github_repo, sha=commit.sha, status="not_found"
92
+ ), None
93
+ except (HttpError, ValueError, KeyError):
94
+ return CommitOutcome(
95
+ report=report, repo=commit.github_repo, sha=commit.sha, status="error"
96
+ ), None
97
+ outcome = CommitOutcome(
98
+ report=report,
99
+ repo=mapping.repo,
100
+ sha=commit.sha,
101
+ status="mapped",
102
+ files_mapped=len(mapping.files),
103
+ computed_diffs=sum(f.diff_source == "computed" for f in mapping.files),
104
+ parse_errors=sum(f.mapping.parse_errors for f in mapping.files),
105
+ truncated=mapping.truncated,
106
+ )
107
+ return outcome, mapping
108
+
109
+
110
+ def run(
111
+ github: GitHubClient, reports: Sequence[dunlap.DunlapReport], as_of: date
112
+ ) -> BaselineReport:
113
+ truth = {r.id: set(r.symbols) for r in reports}
114
+ released_all = {r.id: dunlap.released_candidates(r, filtered=False) for r in reports}
115
+ released = {r.id: dunlap.released_candidates(r, filtered=True) for r in reports}
116
+
117
+ ours: dict[str, set[str]] = {}
118
+ outcomes: list[CommitOutcome] = []
119
+ for report in reports:
120
+ names: set[str] = set()
121
+ for commit in report.commits:
122
+ outcome, mapping = _map(github, report.id, commit)
123
+ outcomes.append(outcome)
124
+ if mapping is not None:
125
+ names |= {
126
+ c.name
127
+ for f in mapping.files
128
+ for c in f.mapping.changed
129
+ if c.kind in ("function", "method")
130
+ }
131
+ ours[report.id] = names
132
+
133
+ units, vulnerable = dunlap.paper_unit_counts(reports)
134
+ symbols_total = sum(len(s) for s in truth.values())
135
+ comparisons = tuple(
136
+ ReportComparison(
137
+ id=r.id,
138
+ symbols=tuple(sorted(truth[r.id])),
139
+ ours=tuple(sorted(ours[r.id])),
140
+ released=tuple(sorted(released[r.id])),
141
+ jaccard=jaccard(ours[r.id], released[r.id]),
142
+ )
143
+ for r in reports
144
+ )
145
+ return BaselineReport(
146
+ as_of=as_of,
147
+ reports=len(reports),
148
+ commits=len(outcomes),
149
+ unique_commits=len({o.sha for o in outcomes}),
150
+ truth_symbols=symbols_total,
151
+ paper={
152
+ "modified_functions": dunlap.PAPER_MODIFIED_FUNCTIONS,
153
+ "vulnerable_functions": dunlap.PAPER_VULNERABLE_FUNCTIONS,
154
+ "missed_functions": dunlap.PAPER_MISSED_FUNCTIONS,
155
+ "precision": dunlap.PAPER_PRECISION,
156
+ "recall": dunlap.PAPER_RECALL,
157
+ "f1": dunlap.PAPER_F1,
158
+ },
159
+ paper_unit_released={
160
+ "units": units,
161
+ "vulnerable_units": vulnerable,
162
+ "precision": vulnerable / units if units else 0.0,
163
+ },
164
+ released_unfiltered=score(released_all, truth, BOOTSTRAP, SEED),
165
+ released_filtered=score(released, truth, BOOTSTRAP, SEED),
166
+ ours=score(ours, truth, BOOTSTRAP, SEED),
167
+ commit_status=dict(sorted(Counter(o.status for o in outcomes).items())),
168
+ computed_diffs=sum(o.computed_diffs for o in outcomes),
169
+ parse_errors=sum(o.parse_errors for o in outcomes),
170
+ truncated_commits=sum(o.truncated for o in outcomes),
171
+ candidates_ours=sum(len(s) for s in ours.values()),
172
+ candidates_released=sum(len(s) for s in released.values()),
173
+ identical_reports=sum(ours[r.id] == released[r.id] for r in reports),
174
+ mean_jaccard=sum(c.jaccard for c in comparisons) / len(comparisons) if comparisons else 0.0,
175
+ released_found=sum(len(released[r.id] & ours[r.id]) for r in reports),
176
+ superset_reports=sum(released[r.id] <= ours[r.id] for r in reports),
177
+ extra_candidates=sum(len(ours[r.id] - released[r.id]) for r in reports),
178
+ extra_in_top5_reports=sum(
179
+ sorted((len(ours[r.id] - released[r.id]) for r in reports), reverse=True)[:5]
180
+ ),
181
+ comparisons=comparisons,
182
+ commit_outcomes=tuple(outcomes),
183
+ )
184
+
185
+
186
+ def _scores_row(label: str, s: SetScores) -> str:
187
+ def ci(interval: Interval | None) -> str:
188
+ return "" if interval is None else f" ({interval.low:.2f} to {interval.high:.2f})"
189
+
190
+ return (
191
+ f"| {label} | {s.true_positives + s.false_positives:,} | {s.true_positives:,} "
192
+ f"| {s.precision:.3f}{ci(s.precision_ci95)} | {s.recall:.3f}{ci(s.recall_ci95)} "
193
+ f"| {s.f1:.3f}{ci(s.f1_ci95)} | {s.miss_rate:.1%} |"
194
+ )
195
+
196
+
197
+ def render_markdown(report: BaselineReport, source_lines: Sequence[str]) -> str:
198
+ p = report.paper
199
+ pu = report.paper_unit_released
200
+ return "\n".join(
201
+ [
202
+ "# ChangedFuncs baseline on the Dunlap Go benchmark",
203
+ "",
204
+ "Generated by `python -m patchsight.pipeline dunlap-baseline`; do not edit by hand. "
205
+ "ChangedFuncs marks every function a fix commit changes as vulnerable. No LLM is "
206
+ "involved; this checks the harness before any model is added.",
207
+ "",
208
+ "## Sources",
209
+ "",
210
+ *source_lines,
211
+ "",
212
+ 'Benchmark data: Dunlap et al., "Pairing Security Advisories with Vulnerable '
213
+ 'Functions Using Open-Source LLMs", DIMVA 2024, released under Apache-2.0. Ground '
214
+ "truth: GoVulnDB symbols as of 2023-10-26 (CC BY 4.0). This report holds counts and "
215
+ "function names only; no source code.",
216
+ "",
217
+ "## Results",
218
+ "",
219
+ f"{report.reports} reports, {report.unique_commits} fix commits "
220
+ f"({report.commits} report-commit pairs), {report.truth_symbols} "
221
+ "true vulnerable symbols. The unit is a (report, function) pair, so a function "
222
+ "changed by two commits of one report counts once. CIs are 95%, from "
223
+ f"{BOOTSTRAP:,} bootstrap resamples of reports.",
224
+ "",
225
+ "| ChangedFuncs on | Candidates | True positives | Precision | Recall | F1 "
226
+ "| Miss rate |",
227
+ "|---|---|---|---|---|---|---|",
228
+ f"| Paper, as published | {p['modified_functions']:,} | {p['vulnerable_functions']:,} "
229
+ f"| {p['precision']:.2f} | {p['recall']:.2f} | {p['f1']:.2f} | not reported |",
230
+ _scores_row("Released data, all paths", report.released_unfiltered),
231
+ _scores_row("Released data, paper's test rule", report.released_filtered),
232
+ _scores_row("**PatchSight pipeline**, paper's test rule", report.ours),
233
+ "",
234
+ "Miss rate is the share of reports where no true symbol was among the candidates.",
235
+ "",
236
+ "## Agreement with the released candidates",
237
+ "",
238
+ f"- Candidates: PatchSight {report.candidates_ours:,}, released data "
239
+ f"{report.candidates_released:,} (both with the paper's test rule).",
240
+ f"- Reports with identical candidate sets: {report.identical_reports} of "
241
+ f"{report.reports}; mean per-report Jaccard {report.mean_jaccard:.3f}.",
242
+ f"- Commits: {', '.join(f'{k} {v}' for k, v in report.commit_status.items())}. "
243
+ f"Diffs computed locally because GitHub omitted the patch: {report.computed_diffs}. "
244
+ f"Files with tree-sitter parse errors: {report.parse_errors}. Truncated commits: "
245
+ f"{report.truncated_commits}.",
246
+ "",
247
+ "## Why the released CSV scores differently",
248
+ "",
249
+ f"The paper reports {p['modified_functions']:,} candidates, "
250
+ f"{p['vulnerable_functions']} of them vulnerable. PatchSight's pipeline, run on the "
251
+ f"same commits, finds {report.candidates_ours:,} with {report.ours.true_positives} "
252
+ "vulnerable, which reproduces the published precision, recall and F1. The authors' "
253
+ f"released CSV holds only {report.candidates_released:,} candidates:",
254
+ "",
255
+ f"- PatchSight finds {report.released_found:,} of the CSV's "
256
+ f"{report.candidates_released:,} candidates "
257
+ f"({report.released_found / max(report.candidates_released, 1):.1%}), and is a "
258
+ f"superset of the CSV in {report.superset_reports} of {report.reports} reports.",
259
+ f"- The CSV lacks {report.extra_candidates:,} of PatchSight's candidates; "
260
+ f"{report.extra_in_top5_reports:,} of them come from just 5 reports with very "
261
+ "large fixes.",
262
+ '- Under the unit the authors\' evaluation code uses (drop paths containing "test", '
263
+ f"group by report, commit and function) the CSV gives {pu['units']:,} units, "
264
+ f"{pu['vulnerable_units']} vulnerable: precision {pu['precision']:.3f}.",
265
+ "",
266
+ "This is consistent with the released CSV being a subset of the paper's candidates, "
267
+ "most likely without its largest fixes. Comparisons with the published numbers "
268
+ "therefore use PatchSight's own candidate sets, not the CSV.",
269
+ "",
270
+ "The precision interval is wide because those few very large fixes contribute most "
271
+ "false positives. Comparisons between two systems should use a paired bootstrap "
272
+ "over the same report resamples.",
273
+ "",
274
+ ]
275
+ )
@@ -0,0 +1,135 @@
1
+ """The Dunlap et al. (DIMVA 2024) Go benchmark, from the authors' released data.
2
+
3
+ Source: https://github.com/s3c2/llm-vulnerable-functions (Apache-2.0), file
4
+ ``data/patchparser-data.tar.gz`` → ``patchparser-data-2023-10-31.csv``. One row per git
5
+ hunk and changed function, for 280 GoVulnDB reports and their 298 fix commits. The row's
6
+ ``symbol_name`` column holds the report's vulnerable symbols from GoVulnDB (cloned
7
+ 2023-10-26), which are the ground truth.
8
+
9
+ The paper excludes "files containing the word 'test'" (§3), which its code implements as a
10
+ case-insensitive substring match on the path. ``paper_path_filter`` reproduces that rule.
11
+ """
12
+
13
+ import ast
14
+ import csv
15
+ from collections.abc import Iterable
16
+ from pathlib import Path
17
+
18
+ from pydantic import BaseModel, ConfigDict
19
+
20
+ DUNLAP_REPO_URL = "https://github.com/s3c2/llm-vulnerable-functions.git"
21
+ DUNLAP_CSV = "data/patchparser-data/patchparser-data-2023-10-31.csv"
22
+
23
+ # Published in the paper, §5.1 and §5.2, for ChangedFuncs on these 280 reports.
24
+ PAPER_MODIFIED_FUNCTIONS = 2370
25
+ PAPER_VULNERABLE_FUNCTIONS = 528
26
+ PAPER_MISSED_FUNCTIONS = 61
27
+ PAPER_PRECISION = 0.22
28
+ PAPER_RECALL = 0.90
29
+ PAPER_F1 = 0.35
30
+ # Best open model in the paper: Mixtral 8x7B.
31
+ PAPER_BEST_PRECISION = 0.60
32
+ PAPER_BEST_RECALL = 0.74
33
+ PAPER_BEST_F1 = 0.66
34
+
35
+
36
+ class FixCommit(BaseModel):
37
+ model_config = ConfigDict(frozen=True)
38
+
39
+ github_repo: str
40
+ sha: str
41
+ googlesource: bool
42
+
43
+
44
+ class ReleasedCandidate(BaseModel):
45
+ """One changed function in the authors' released data."""
46
+
47
+ model_config = ConfigDict(frozen=True)
48
+
49
+ sha: str
50
+ path: str
51
+ function: str
52
+ vulnerable: bool
53
+
54
+
55
+ class DunlapReport(BaseModel):
56
+ model_config = ConfigDict(frozen=True)
57
+
58
+ id: str
59
+ commits: tuple[FixCommit, ...]
60
+ symbols: frozenset[str]
61
+ released: tuple[ReleasedCandidate, ...]
62
+
63
+
64
+ def paper_path_filter(path: str) -> bool:
65
+ """True if the paper keeps this path: Go source whose path does not contain "test"."""
66
+ return path.endswith(".go") and "test" not in path.lower()
67
+
68
+
69
+ def github_repo(owner: str, name: str, repo_type: str) -> str:
70
+ """Where a fix commit lives on GitHub. Go's own repos are mirrored under ``golang``."""
71
+ if repo_type == "GoogleSource":
72
+ return f"golang/{name}".lower()
73
+ return f"{owner}/{name}".lower()
74
+
75
+
76
+ def _one(value: str) -> str:
77
+ items = ast.literal_eval(value)
78
+ if not isinstance(items, list) or len(items) != 1:
79
+ raise ValueError(f"expected a one-element list, got {value[:60]!r}")
80
+ return str(items[0])
81
+
82
+
83
+ def load(csv_path: Path) -> list[DunlapReport]:
84
+ """All reports in the released data, sorted by ID."""
85
+ csv.field_size_limit(1 << 30)
86
+ commits: dict[str, dict[tuple[str, str], FixCommit]] = {}
87
+ symbols: dict[str, set[str]] = {}
88
+ released: dict[str, dict[tuple[str, str, str], bool]] = {}
89
+ with csv_path.open(newline="", encoding="utf-8") as handle:
90
+ for row in csv.DictReader(handle):
91
+ report = row["id"]
92
+ repo = github_repo(row["repo_owner"], row["repo_name"], row["repo_type"])
93
+ commit = FixCommit(
94
+ github_repo=repo,
95
+ sha=row["vfc_sha"],
96
+ googlesource=row["repo_type"] == "GoogleSource",
97
+ )
98
+ commits.setdefault(report, {})[(repo, commit.sha)] = commit
99
+ symbols.setdefault(report, set()).update(ast.literal_eval(row["symbol_name"]))
100
+ key = (commit.sha, row["file_name"], _one(row["changed_combined_functions"]))
101
+ label = _one(row["label_combined_unique"]) == "True"
102
+ seen = released.setdefault(report, {})
103
+ seen[key] = seen.get(key, False) or label
104
+ return [
105
+ DunlapReport(
106
+ id=report,
107
+ commits=tuple(commits[report][k] for k in sorted(commits[report])),
108
+ symbols=frozenset(symbols[report]),
109
+ released=tuple(
110
+ ReleasedCandidate(sha=sha, path=path, function=function, vulnerable=label)
111
+ for (sha, path, function), label in sorted(released[report].items())
112
+ ),
113
+ )
114
+ for report in sorted(commits)
115
+ ]
116
+
117
+
118
+ def released_candidates(report: DunlapReport, filtered: bool) -> set[str]:
119
+ """The report's changed function names in the released data."""
120
+ return {c.function for c in report.released if not filtered or paper_path_filter(c.path)}
121
+
122
+
123
+ def paper_unit_counts(reports: Iterable[DunlapReport]) -> tuple[int, int]:
124
+ """(units, vulnerable units) under the paper code's unit: (report, commit, function).
125
+
126
+ The authors' evaluation code drops paths containing "test" and then groups rows by
127
+ report, commit and function. This counts the same units from the released data.
128
+ """
129
+ units: dict[tuple[str, str, str], bool] = {}
130
+ for report in reports:
131
+ for c in report.released:
132
+ if paper_path_filter(c.path):
133
+ key = (report.id, c.sha, c.function)
134
+ units[key] = units.get(key, False) or c.vulnerable
135
+ return len(units), sum(units.values())