patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Published benchmarks used to evaluate extraction."""
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Benchmark bundles: what a stranger needs to recompute the benchmark scores.
|
|
2
|
+
|
|
3
|
+
The fix diffs cannot be redistributed, because they are other projects' code. The scores do
|
|
4
|
+
not need them, though: they need only which functions each fix changed. A bundle records
|
|
5
|
+
that metadata (commit SHAs, file paths, function names and kinds) together with each report's
|
|
6
|
+
ground-truth symbols, so ``reproduce`` recomputes the tables offline, with no API key.
|
|
7
|
+
|
|
8
|
+
Contents and licences: candidate metadata derived by PatchSight from public fix commits;
|
|
9
|
+
ground-truth symbols from GoVulnDB as released by Dunlap et al. (GoVulnDB is CC BY 4.0; the
|
|
10
|
+
Dunlap et al. data is Apache-2.0).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from pydantic import BaseModel, ConfigDict
|
|
18
|
+
|
|
19
|
+
from patchsight.eval.metrics import SetScores, score
|
|
20
|
+
from patchsight.pipeline.benchmarks import dunlap
|
|
21
|
+
from patchsight.pipeline.diffs.commits import map_commit
|
|
22
|
+
from patchsight.pipeline.diffs.github import GitHubClient
|
|
23
|
+
|
|
24
|
+
BUNDLE_VERSION = 1
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class BundleCandidate(BaseModel):
|
|
28
|
+
model_config = ConfigDict(frozen=True)
|
|
29
|
+
|
|
30
|
+
commit: str
|
|
31
|
+
path: str
|
|
32
|
+
name: str
|
|
33
|
+
kind: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class BundleReport(BaseModel):
|
|
37
|
+
model_config = ConfigDict(frozen=True)
|
|
38
|
+
|
|
39
|
+
id: str
|
|
40
|
+
symbols: tuple[str, ...]
|
|
41
|
+
commits: tuple[str, ...]
|
|
42
|
+
candidates: tuple[BundleCandidate, ...]
|
|
43
|
+
|
|
44
|
+
def changed_functions(self) -> set[str]:
|
|
45
|
+
"""ChangedFuncs' prediction: every changed function or method, by name."""
|
|
46
|
+
return {c.name for c in self.candidates if c.kind in ("function", "method")}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Bundle(BaseModel):
|
|
50
|
+
bundle_version: int
|
|
51
|
+
benchmark: str
|
|
52
|
+
source: dict[str, str]
|
|
53
|
+
path_rule: str
|
|
54
|
+
reports: tuple[BundleReport, ...]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def export(
|
|
58
|
+
github: GitHubClient, reports: Sequence[dunlap.DunlapReport], source: dict[str, str]
|
|
59
|
+
) -> Bundle:
|
|
60
|
+
"""Map every report's fix commits (from the response cache when present)."""
|
|
61
|
+
exported = []
|
|
62
|
+
for report in reports:
|
|
63
|
+
candidates: list[BundleCandidate] = []
|
|
64
|
+
for commit in report.commits:
|
|
65
|
+
mapping = map_commit(
|
|
66
|
+
github, commit.github_repo, commit.sha, include=dunlap.paper_path_filter
|
|
67
|
+
)
|
|
68
|
+
candidates.extend(
|
|
69
|
+
BundleCandidate(commit=commit.sha, path=f.mapping.path, name=c.name, kind=c.kind)
|
|
70
|
+
for f in mapping.files
|
|
71
|
+
for c in f.mapping.changed
|
|
72
|
+
)
|
|
73
|
+
exported.append(
|
|
74
|
+
BundleReport(
|
|
75
|
+
id=report.id,
|
|
76
|
+
symbols=tuple(sorted(report.symbols)),
|
|
77
|
+
commits=tuple(c.sha for c in report.commits),
|
|
78
|
+
candidates=tuple(sorted(candidates, key=lambda c: (c.commit, c.path, c.name))),
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
return Bundle(
|
|
82
|
+
bundle_version=BUNDLE_VERSION,
|
|
83
|
+
benchmark="dunlap-dimva24",
|
|
84
|
+
source=source,
|
|
85
|
+
path_rule='Go files whose path does not contain "test" (the paper\'s rule)',
|
|
86
|
+
reports=tuple(exported),
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def save(bundle: Bundle, path: Path) -> None:
|
|
91
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
path.write_text(json.dumps(bundle.model_dump(mode="json"), indent=1) + "\n", encoding="utf-8")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def load(path: Path) -> Bundle:
|
|
96
|
+
bundle = Bundle.model_validate_json(path.read_text(encoding="utf-8"))
|
|
97
|
+
if bundle.bundle_version != BUNDLE_VERSION:
|
|
98
|
+
raise ValueError(f"{path} is bundle version {bundle.bundle_version}, not {BUNDLE_VERSION}")
|
|
99
|
+
return bundle
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def changedfuncs_scores(
|
|
103
|
+
bundle: Bundle, ids: frozenset[str] | None = None, bootstrap: int = 0, seed: int = 0
|
|
104
|
+
) -> SetScores:
|
|
105
|
+
chosen = [r for r in bundle.reports if ids is None or r.id in ids]
|
|
106
|
+
predicted = {r.id: r.changed_functions() for r in chosen}
|
|
107
|
+
truth = {r.id: set(r.symbols) for r in chosen}
|
|
108
|
+
return score(predicted, truth, bootstrap, seed)
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Reproduce the ChangedFuncs baseline on the Dunlap benchmark (proposal §15, Phase 1 step 3).
|
|
2
|
+
|
|
3
|
+
ChangedFuncs marks every function a fix commit changes as vulnerable. Running it before
|
|
4
|
+
any LLM proves the harness: diff fetching, hunk mapping and scoring. It is scored three ways:
|
|
5
|
+
|
|
6
|
+
1. the paper's published numbers, as reported;
|
|
7
|
+
2. the authors' released candidate lists, scored by PatchSight's metric code;
|
|
8
|
+
3. PatchSight's own pipeline: the same 298 commits fetched from GitHub and mapped with
|
|
9
|
+
PatchSight's tree-sitter code, using the paper's path rule.
|
|
10
|
+
|
|
11
|
+
Agreement between 2 and 3 is the direct check on the diff and mapping code.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from collections import Counter
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
from datetime import date
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict
|
|
19
|
+
|
|
20
|
+
from patchsight.eval.metrics import Interval, SetScores, jaccard, score
|
|
21
|
+
from patchsight.pipeline.benchmarks import dunlap
|
|
22
|
+
from patchsight.pipeline.diffs.commits import CommitMapping, map_commit
|
|
23
|
+
from patchsight.pipeline.diffs.github import GitHubClient
|
|
24
|
+
from patchsight.pipeline.http import HttpError, NotFoundError, RateLimitError
|
|
25
|
+
|
|
26
|
+
BOOTSTRAP = 2000
|
|
27
|
+
SEED = 20261114
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CommitOutcome(BaseModel):
|
|
31
|
+
model_config = ConfigDict(frozen=True)
|
|
32
|
+
|
|
33
|
+
report: str
|
|
34
|
+
repo: str
|
|
35
|
+
sha: str
|
|
36
|
+
status: str
|
|
37
|
+
files_mapped: int = 0
|
|
38
|
+
computed_diffs: int = 0
|
|
39
|
+
parse_errors: int = 0
|
|
40
|
+
truncated: bool = False
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ReportComparison(BaseModel):
|
|
44
|
+
model_config = ConfigDict(frozen=True)
|
|
45
|
+
|
|
46
|
+
id: str
|
|
47
|
+
symbols: tuple[str, ...]
|
|
48
|
+
ours: tuple[str, ...]
|
|
49
|
+
released: tuple[str, ...]
|
|
50
|
+
jaccard: float
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class BaselineReport(BaseModel):
|
|
54
|
+
as_of: date
|
|
55
|
+
reports: int
|
|
56
|
+
commits: int
|
|
57
|
+
unique_commits: int
|
|
58
|
+
truth_symbols: int
|
|
59
|
+
paper: dict[str, float | int]
|
|
60
|
+
paper_unit_released: dict[str, float | int]
|
|
61
|
+
released_unfiltered: SetScores
|
|
62
|
+
released_filtered: SetScores
|
|
63
|
+
ours: SetScores
|
|
64
|
+
commit_status: dict[str, int]
|
|
65
|
+
computed_diffs: int
|
|
66
|
+
parse_errors: int
|
|
67
|
+
truncated_commits: int
|
|
68
|
+
candidates_ours: int
|
|
69
|
+
candidates_released: int
|
|
70
|
+
identical_reports: int
|
|
71
|
+
mean_jaccard: float
|
|
72
|
+
released_found: int
|
|
73
|
+
superset_reports: int
|
|
74
|
+
extra_candidates: int
|
|
75
|
+
extra_in_top5_reports: int
|
|
76
|
+
comparisons: tuple[ReportComparison, ...]
|
|
77
|
+
commit_outcomes: tuple[CommitOutcome, ...]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _map(
|
|
81
|
+
github: GitHubClient, report: str, commit: dunlap.FixCommit
|
|
82
|
+
) -> tuple[CommitOutcome, CommitMapping | None]:
|
|
83
|
+
try:
|
|
84
|
+
mapping = map_commit(
|
|
85
|
+
github, commit.github_repo, commit.sha, include=dunlap.paper_path_filter
|
|
86
|
+
)
|
|
87
|
+
except RateLimitError:
|
|
88
|
+
raise
|
|
89
|
+
except NotFoundError:
|
|
90
|
+
return CommitOutcome(
|
|
91
|
+
report=report, repo=commit.github_repo, sha=commit.sha, status="not_found"
|
|
92
|
+
), None
|
|
93
|
+
except (HttpError, ValueError, KeyError):
|
|
94
|
+
return CommitOutcome(
|
|
95
|
+
report=report, repo=commit.github_repo, sha=commit.sha, status="error"
|
|
96
|
+
), None
|
|
97
|
+
outcome = CommitOutcome(
|
|
98
|
+
report=report,
|
|
99
|
+
repo=mapping.repo,
|
|
100
|
+
sha=commit.sha,
|
|
101
|
+
status="mapped",
|
|
102
|
+
files_mapped=len(mapping.files),
|
|
103
|
+
computed_diffs=sum(f.diff_source == "computed" for f in mapping.files),
|
|
104
|
+
parse_errors=sum(f.mapping.parse_errors for f in mapping.files),
|
|
105
|
+
truncated=mapping.truncated,
|
|
106
|
+
)
|
|
107
|
+
return outcome, mapping
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def run(
|
|
111
|
+
github: GitHubClient, reports: Sequence[dunlap.DunlapReport], as_of: date
|
|
112
|
+
) -> BaselineReport:
|
|
113
|
+
truth = {r.id: set(r.symbols) for r in reports}
|
|
114
|
+
released_all = {r.id: dunlap.released_candidates(r, filtered=False) for r in reports}
|
|
115
|
+
released = {r.id: dunlap.released_candidates(r, filtered=True) for r in reports}
|
|
116
|
+
|
|
117
|
+
ours: dict[str, set[str]] = {}
|
|
118
|
+
outcomes: list[CommitOutcome] = []
|
|
119
|
+
for report in reports:
|
|
120
|
+
names: set[str] = set()
|
|
121
|
+
for commit in report.commits:
|
|
122
|
+
outcome, mapping = _map(github, report.id, commit)
|
|
123
|
+
outcomes.append(outcome)
|
|
124
|
+
if mapping is not None:
|
|
125
|
+
names |= {
|
|
126
|
+
c.name
|
|
127
|
+
for f in mapping.files
|
|
128
|
+
for c in f.mapping.changed
|
|
129
|
+
if c.kind in ("function", "method")
|
|
130
|
+
}
|
|
131
|
+
ours[report.id] = names
|
|
132
|
+
|
|
133
|
+
units, vulnerable = dunlap.paper_unit_counts(reports)
|
|
134
|
+
symbols_total = sum(len(s) for s in truth.values())
|
|
135
|
+
comparisons = tuple(
|
|
136
|
+
ReportComparison(
|
|
137
|
+
id=r.id,
|
|
138
|
+
symbols=tuple(sorted(truth[r.id])),
|
|
139
|
+
ours=tuple(sorted(ours[r.id])),
|
|
140
|
+
released=tuple(sorted(released[r.id])),
|
|
141
|
+
jaccard=jaccard(ours[r.id], released[r.id]),
|
|
142
|
+
)
|
|
143
|
+
for r in reports
|
|
144
|
+
)
|
|
145
|
+
return BaselineReport(
|
|
146
|
+
as_of=as_of,
|
|
147
|
+
reports=len(reports),
|
|
148
|
+
commits=len(outcomes),
|
|
149
|
+
unique_commits=len({o.sha for o in outcomes}),
|
|
150
|
+
truth_symbols=symbols_total,
|
|
151
|
+
paper={
|
|
152
|
+
"modified_functions": dunlap.PAPER_MODIFIED_FUNCTIONS,
|
|
153
|
+
"vulnerable_functions": dunlap.PAPER_VULNERABLE_FUNCTIONS,
|
|
154
|
+
"missed_functions": dunlap.PAPER_MISSED_FUNCTIONS,
|
|
155
|
+
"precision": dunlap.PAPER_PRECISION,
|
|
156
|
+
"recall": dunlap.PAPER_RECALL,
|
|
157
|
+
"f1": dunlap.PAPER_F1,
|
|
158
|
+
},
|
|
159
|
+
paper_unit_released={
|
|
160
|
+
"units": units,
|
|
161
|
+
"vulnerable_units": vulnerable,
|
|
162
|
+
"precision": vulnerable / units if units else 0.0,
|
|
163
|
+
},
|
|
164
|
+
released_unfiltered=score(released_all, truth, BOOTSTRAP, SEED),
|
|
165
|
+
released_filtered=score(released, truth, BOOTSTRAP, SEED),
|
|
166
|
+
ours=score(ours, truth, BOOTSTRAP, SEED),
|
|
167
|
+
commit_status=dict(sorted(Counter(o.status for o in outcomes).items())),
|
|
168
|
+
computed_diffs=sum(o.computed_diffs for o in outcomes),
|
|
169
|
+
parse_errors=sum(o.parse_errors for o in outcomes),
|
|
170
|
+
truncated_commits=sum(o.truncated for o in outcomes),
|
|
171
|
+
candidates_ours=sum(len(s) for s in ours.values()),
|
|
172
|
+
candidates_released=sum(len(s) for s in released.values()),
|
|
173
|
+
identical_reports=sum(ours[r.id] == released[r.id] for r in reports),
|
|
174
|
+
mean_jaccard=sum(c.jaccard for c in comparisons) / len(comparisons) if comparisons else 0.0,
|
|
175
|
+
released_found=sum(len(released[r.id] & ours[r.id]) for r in reports),
|
|
176
|
+
superset_reports=sum(released[r.id] <= ours[r.id] for r in reports),
|
|
177
|
+
extra_candidates=sum(len(ours[r.id] - released[r.id]) for r in reports),
|
|
178
|
+
extra_in_top5_reports=sum(
|
|
179
|
+
sorted((len(ours[r.id] - released[r.id]) for r in reports), reverse=True)[:5]
|
|
180
|
+
),
|
|
181
|
+
comparisons=comparisons,
|
|
182
|
+
commit_outcomes=tuple(outcomes),
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _scores_row(label: str, s: SetScores) -> str:
|
|
187
|
+
def ci(interval: Interval | None) -> str:
|
|
188
|
+
return "" if interval is None else f" ({interval.low:.2f} to {interval.high:.2f})"
|
|
189
|
+
|
|
190
|
+
return (
|
|
191
|
+
f"| {label} | {s.true_positives + s.false_positives:,} | {s.true_positives:,} "
|
|
192
|
+
f"| {s.precision:.3f}{ci(s.precision_ci95)} | {s.recall:.3f}{ci(s.recall_ci95)} "
|
|
193
|
+
f"| {s.f1:.3f}{ci(s.f1_ci95)} | {s.miss_rate:.1%} |"
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def render_markdown(report: BaselineReport, source_lines: Sequence[str]) -> str:
|
|
198
|
+
p = report.paper
|
|
199
|
+
pu = report.paper_unit_released
|
|
200
|
+
return "\n".join(
|
|
201
|
+
[
|
|
202
|
+
"# ChangedFuncs baseline on the Dunlap Go benchmark",
|
|
203
|
+
"",
|
|
204
|
+
"Generated by `python -m patchsight.pipeline dunlap-baseline`; do not edit by hand. "
|
|
205
|
+
"ChangedFuncs marks every function a fix commit changes as vulnerable. No LLM is "
|
|
206
|
+
"involved; this checks the harness before any model is added.",
|
|
207
|
+
"",
|
|
208
|
+
"## Sources",
|
|
209
|
+
"",
|
|
210
|
+
*source_lines,
|
|
211
|
+
"",
|
|
212
|
+
'Benchmark data: Dunlap et al., "Pairing Security Advisories with Vulnerable '
|
|
213
|
+
'Functions Using Open-Source LLMs", DIMVA 2024, released under Apache-2.0. Ground '
|
|
214
|
+
"truth: GoVulnDB symbols as of 2023-10-26 (CC BY 4.0). This report holds counts and "
|
|
215
|
+
"function names only; no source code.",
|
|
216
|
+
"",
|
|
217
|
+
"## Results",
|
|
218
|
+
"",
|
|
219
|
+
f"{report.reports} reports, {report.unique_commits} fix commits "
|
|
220
|
+
f"({report.commits} report-commit pairs), {report.truth_symbols} "
|
|
221
|
+
"true vulnerable symbols. The unit is a (report, function) pair, so a function "
|
|
222
|
+
"changed by two commits of one report counts once. CIs are 95%, from "
|
|
223
|
+
f"{BOOTSTRAP:,} bootstrap resamples of reports.",
|
|
224
|
+
"",
|
|
225
|
+
"| ChangedFuncs on | Candidates | True positives | Precision | Recall | F1 "
|
|
226
|
+
"| Miss rate |",
|
|
227
|
+
"|---|---|---|---|---|---|---|",
|
|
228
|
+
f"| Paper, as published | {p['modified_functions']:,} | {p['vulnerable_functions']:,} "
|
|
229
|
+
f"| {p['precision']:.2f} | {p['recall']:.2f} | {p['f1']:.2f} | not reported |",
|
|
230
|
+
_scores_row("Released data, all paths", report.released_unfiltered),
|
|
231
|
+
_scores_row("Released data, paper's test rule", report.released_filtered),
|
|
232
|
+
_scores_row("**PatchSight pipeline**, paper's test rule", report.ours),
|
|
233
|
+
"",
|
|
234
|
+
"Miss rate is the share of reports where no true symbol was among the candidates.",
|
|
235
|
+
"",
|
|
236
|
+
"## Agreement with the released candidates",
|
|
237
|
+
"",
|
|
238
|
+
f"- Candidates: PatchSight {report.candidates_ours:,}, released data "
|
|
239
|
+
f"{report.candidates_released:,} (both with the paper's test rule).",
|
|
240
|
+
f"- Reports with identical candidate sets: {report.identical_reports} of "
|
|
241
|
+
f"{report.reports}; mean per-report Jaccard {report.mean_jaccard:.3f}.",
|
|
242
|
+
f"- Commits: {', '.join(f'{k} {v}' for k, v in report.commit_status.items())}. "
|
|
243
|
+
f"Diffs computed locally because GitHub omitted the patch: {report.computed_diffs}. "
|
|
244
|
+
f"Files with tree-sitter parse errors: {report.parse_errors}. Truncated commits: "
|
|
245
|
+
f"{report.truncated_commits}.",
|
|
246
|
+
"",
|
|
247
|
+
"## Why the released CSV scores differently",
|
|
248
|
+
"",
|
|
249
|
+
f"The paper reports {p['modified_functions']:,} candidates, "
|
|
250
|
+
f"{p['vulnerable_functions']} of them vulnerable. PatchSight's pipeline, run on the "
|
|
251
|
+
f"same commits, finds {report.candidates_ours:,} with {report.ours.true_positives} "
|
|
252
|
+
"vulnerable, which reproduces the published precision, recall and F1. The authors' "
|
|
253
|
+
f"released CSV holds only {report.candidates_released:,} candidates:",
|
|
254
|
+
"",
|
|
255
|
+
f"- PatchSight finds {report.released_found:,} of the CSV's "
|
|
256
|
+
f"{report.candidates_released:,} candidates "
|
|
257
|
+
f"({report.released_found / max(report.candidates_released, 1):.1%}), and is a "
|
|
258
|
+
f"superset of the CSV in {report.superset_reports} of {report.reports} reports.",
|
|
259
|
+
f"- The CSV lacks {report.extra_candidates:,} of PatchSight's candidates; "
|
|
260
|
+
f"{report.extra_in_top5_reports:,} of them come from just 5 reports with very "
|
|
261
|
+
"large fixes.",
|
|
262
|
+
'- Under the unit the authors\' evaluation code uses (drop paths containing "test", '
|
|
263
|
+
f"group by report, commit and function) the CSV gives {pu['units']:,} units, "
|
|
264
|
+
f"{pu['vulnerable_units']} vulnerable: precision {pu['precision']:.3f}.",
|
|
265
|
+
"",
|
|
266
|
+
"This is consistent with the released CSV being a subset of the paper's candidates, "
|
|
267
|
+
"most likely without its largest fixes. Comparisons with the published numbers "
|
|
268
|
+
"therefore use PatchSight's own candidate sets, not the CSV.",
|
|
269
|
+
"",
|
|
270
|
+
"The precision interval is wide because those few very large fixes contribute most "
|
|
271
|
+
"false positives. Comparisons between two systems should use a paired bootstrap "
|
|
272
|
+
"over the same report resamples.",
|
|
273
|
+
"",
|
|
274
|
+
]
|
|
275
|
+
)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""The Dunlap et al. (DIMVA 2024) Go benchmark, from the authors' released data.
|
|
2
|
+
|
|
3
|
+
Source: https://github.com/s3c2/llm-vulnerable-functions (Apache-2.0), file
|
|
4
|
+
``data/patchparser-data.tar.gz`` → ``patchparser-data-2023-10-31.csv``. One row per git
|
|
5
|
+
hunk and changed function, for 280 GoVulnDB reports and their 298 fix commits. The row's
|
|
6
|
+
``symbol_name`` column holds the report's vulnerable symbols from GoVulnDB (cloned
|
|
7
|
+
2023-10-26), which are the ground truth.
|
|
8
|
+
|
|
9
|
+
The paper excludes "files containing the word 'test'" (§3), which its code implements as a
|
|
10
|
+
case-insensitive substring match on the path. ``paper_path_filter`` reproduces that rule.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
import csv
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict
|
|
19
|
+
|
|
20
|
+
DUNLAP_REPO_URL = "https://github.com/s3c2/llm-vulnerable-functions.git"
|
|
21
|
+
DUNLAP_CSV = "data/patchparser-data/patchparser-data-2023-10-31.csv"
|
|
22
|
+
|
|
23
|
+
# Published in the paper, §5.1 and §5.2, for ChangedFuncs on these 280 reports.
|
|
24
|
+
PAPER_MODIFIED_FUNCTIONS = 2370
|
|
25
|
+
PAPER_VULNERABLE_FUNCTIONS = 528
|
|
26
|
+
PAPER_MISSED_FUNCTIONS = 61
|
|
27
|
+
PAPER_PRECISION = 0.22
|
|
28
|
+
PAPER_RECALL = 0.90
|
|
29
|
+
PAPER_F1 = 0.35
|
|
30
|
+
# Best open model in the paper: Mixtral 8x7B.
|
|
31
|
+
PAPER_BEST_PRECISION = 0.60
|
|
32
|
+
PAPER_BEST_RECALL = 0.74
|
|
33
|
+
PAPER_BEST_F1 = 0.66
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class FixCommit(BaseModel):
|
|
37
|
+
model_config = ConfigDict(frozen=True)
|
|
38
|
+
|
|
39
|
+
github_repo: str
|
|
40
|
+
sha: str
|
|
41
|
+
googlesource: bool
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ReleasedCandidate(BaseModel):
|
|
45
|
+
"""One changed function in the authors' released data."""
|
|
46
|
+
|
|
47
|
+
model_config = ConfigDict(frozen=True)
|
|
48
|
+
|
|
49
|
+
sha: str
|
|
50
|
+
path: str
|
|
51
|
+
function: str
|
|
52
|
+
vulnerable: bool
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DunlapReport(BaseModel):
|
|
56
|
+
model_config = ConfigDict(frozen=True)
|
|
57
|
+
|
|
58
|
+
id: str
|
|
59
|
+
commits: tuple[FixCommit, ...]
|
|
60
|
+
symbols: frozenset[str]
|
|
61
|
+
released: tuple[ReleasedCandidate, ...]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def paper_path_filter(path: str) -> bool:
|
|
65
|
+
"""True if the paper keeps this path: Go source whose path does not contain "test"."""
|
|
66
|
+
return path.endswith(".go") and "test" not in path.lower()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def github_repo(owner: str, name: str, repo_type: str) -> str:
|
|
70
|
+
"""Where a fix commit lives on GitHub. Go's own repos are mirrored under ``golang``."""
|
|
71
|
+
if repo_type == "GoogleSource":
|
|
72
|
+
return f"golang/{name}".lower()
|
|
73
|
+
return f"{owner}/{name}".lower()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _one(value: str) -> str:
|
|
77
|
+
items = ast.literal_eval(value)
|
|
78
|
+
if not isinstance(items, list) or len(items) != 1:
|
|
79
|
+
raise ValueError(f"expected a one-element list, got {value[:60]!r}")
|
|
80
|
+
return str(items[0])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def load(csv_path: Path) -> list[DunlapReport]:
|
|
84
|
+
"""All reports in the released data, sorted by ID."""
|
|
85
|
+
csv.field_size_limit(1 << 30)
|
|
86
|
+
commits: dict[str, dict[tuple[str, str], FixCommit]] = {}
|
|
87
|
+
symbols: dict[str, set[str]] = {}
|
|
88
|
+
released: dict[str, dict[tuple[str, str, str], bool]] = {}
|
|
89
|
+
with csv_path.open(newline="", encoding="utf-8") as handle:
|
|
90
|
+
for row in csv.DictReader(handle):
|
|
91
|
+
report = row["id"]
|
|
92
|
+
repo = github_repo(row["repo_owner"], row["repo_name"], row["repo_type"])
|
|
93
|
+
commit = FixCommit(
|
|
94
|
+
github_repo=repo,
|
|
95
|
+
sha=row["vfc_sha"],
|
|
96
|
+
googlesource=row["repo_type"] == "GoogleSource",
|
|
97
|
+
)
|
|
98
|
+
commits.setdefault(report, {})[(repo, commit.sha)] = commit
|
|
99
|
+
symbols.setdefault(report, set()).update(ast.literal_eval(row["symbol_name"]))
|
|
100
|
+
key = (commit.sha, row["file_name"], _one(row["changed_combined_functions"]))
|
|
101
|
+
label = _one(row["label_combined_unique"]) == "True"
|
|
102
|
+
seen = released.setdefault(report, {})
|
|
103
|
+
seen[key] = seen.get(key, False) or label
|
|
104
|
+
return [
|
|
105
|
+
DunlapReport(
|
|
106
|
+
id=report,
|
|
107
|
+
commits=tuple(commits[report][k] for k in sorted(commits[report])),
|
|
108
|
+
symbols=frozenset(symbols[report]),
|
|
109
|
+
released=tuple(
|
|
110
|
+
ReleasedCandidate(sha=sha, path=path, function=function, vulnerable=label)
|
|
111
|
+
for (sha, path, function), label in sorted(released[report].items())
|
|
112
|
+
),
|
|
113
|
+
)
|
|
114
|
+
for report in sorted(commits)
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def released_candidates(report: DunlapReport, filtered: bool) -> set[str]:
|
|
119
|
+
"""The report's changed function names in the released data."""
|
|
120
|
+
return {c.function for c in report.released if not filtered or paper_path_filter(c.path)}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def paper_unit_counts(reports: Iterable[DunlapReport]) -> tuple[int, int]:
|
|
124
|
+
"""(units, vulnerable units) under the paper code's unit: (report, commit, function).
|
|
125
|
+
|
|
126
|
+
The authors' evaluation code drops paths containing "test" and then groups rows by
|
|
127
|
+
report, commit and function. This counts the same units from the released data.
|
|
128
|
+
"""
|
|
129
|
+
units: dict[tuple[str, str, str], bool] = {}
|
|
130
|
+
for report in reports:
|
|
131
|
+
for c in report.released:
|
|
132
|
+
if paper_path_filter(c.path):
|
|
133
|
+
key = (report.id, c.sha, c.function)
|
|
134
|
+
units[key] = units.get(key, False) or c.vulnerable
|
|
135
|
+
return len(units), sum(units.values())
|