patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Run LLM extraction on the Dunlap Go benchmark and score it against GoVulnDB symbols.
|
|
2
|
+
|
|
3
|
+
Candidates come from PatchSight's own fetch-and-map pipeline, the same candidate sets that
|
|
4
|
+
reproduce the published ChangedFuncs baseline. ChangedFuncs is scored on the same reports,
|
|
5
|
+
so every run carries its own naive baseline.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import csv
|
|
9
|
+
import random
|
|
10
|
+
from collections import Counter
|
|
11
|
+
from collections.abc import Callable, Sequence
|
|
12
|
+
from datetime import date
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel
|
|
16
|
+
|
|
17
|
+
from patchsight.eval.metrics import Interval, SetScores, score
|
|
18
|
+
from patchsight.pipeline.benchmarks import dunlap
|
|
19
|
+
from patchsight.pipeline.diffs.commits import CommitMapping, map_commit
|
|
20
|
+
from patchsight.pipeline.diffs.github import GitHubClient
|
|
21
|
+
from patchsight.pipeline.extract.extractor import ExtractionOutcome, extract
|
|
22
|
+
from patchsight.pipeline.extract.prompt import AdvisoryText, CandidateInput, PromptTemplate
|
|
23
|
+
from patchsight.pipeline.llm.client import LLMClient
|
|
24
|
+
from patchsight.pipeline.llm.config import LLMConfig
|
|
25
|
+
from patchsight.pipeline.llm.tokens import TokenCounter
|
|
26
|
+
|
|
27
|
+
DUNLAP_ADVISORIES = "data/advisory-data/govulndb-2023-10-26.csv"
|
|
28
|
+
BOOTSTRAP = 2000
|
|
29
|
+
SEED = 20261114
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ExtractionRun(BaseModel):
|
|
33
|
+
as_of: date
|
|
34
|
+
model: str
|
|
35
|
+
prompt_version: str
|
|
36
|
+
reports: int
|
|
37
|
+
split: str
|
|
38
|
+
split_sha256: str | None
|
|
39
|
+
sample_size: int | None
|
|
40
|
+
sample_seed: int
|
|
41
|
+
extraction: SetScores
|
|
42
|
+
changedfuncs: SetScores
|
|
43
|
+
status: dict[str, int]
|
|
44
|
+
truncated: int
|
|
45
|
+
invented_ids_first_attempt: int
|
|
46
|
+
entry_points_dropped: int
|
|
47
|
+
prompt_tokens: int
|
|
48
|
+
completion_tokens: int
|
|
49
|
+
estimated_prompt_tokens: int
|
|
50
|
+
calls: int
|
|
51
|
+
cached_calls: int
|
|
52
|
+
provider_rejections: int
|
|
53
|
+
outcomes: tuple[ExtractionOutcome, ...]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def load_advisories(path: Path) -> dict[str, AdvisoryText]:
|
|
57
|
+
csv.field_size_limit(1 << 30)
|
|
58
|
+
with path.open(newline="", encoding="utf-8") as handle:
|
|
59
|
+
return {
|
|
60
|
+
row["id"]: AdvisoryText(id=row["id"], summary=row["summary"], details=row["details"])
|
|
61
|
+
for row in csv.DictReader(handle)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def candidate_inputs(mappings: Sequence[CommitMapping]) -> tuple[list[CandidateInput], set[str]]:
|
|
66
|
+
"""Function and method candidates from mapped commits, and every known definition name."""
|
|
67
|
+
inputs: list[CandidateInput] = []
|
|
68
|
+
known: set[str] = set()
|
|
69
|
+
for mapping in mappings:
|
|
70
|
+
for file in mapping.files:
|
|
71
|
+
known.update(file.mapping.known_names)
|
|
72
|
+
by_index = {h.index: h for h in file.hunks}
|
|
73
|
+
for changed in file.mapping.changed:
|
|
74
|
+
if changed.kind not in ("function", "method"):
|
|
75
|
+
continue
|
|
76
|
+
inputs.append(
|
|
77
|
+
CandidateInput(
|
|
78
|
+
path=file.mapping.path,
|
|
79
|
+
name=changed.name,
|
|
80
|
+
kind=changed.kind,
|
|
81
|
+
hunks=tuple(by_index[i] for i in changed.hunks if i in by_index),
|
|
82
|
+
source=mapping.sha,
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
return inputs, known
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def select(
|
|
89
|
+
reports: Sequence[dunlap.DunlapReport],
|
|
90
|
+
size: int | None,
|
|
91
|
+
seed: int,
|
|
92
|
+
ids: frozenset[str] | None = None,
|
|
93
|
+
) -> list[dunlap.DunlapReport]:
|
|
94
|
+
ordered = sorted((r for r in reports if ids is None or r.id in ids), key=lambda r: r.id)
|
|
95
|
+
if size is None or size >= len(ordered):
|
|
96
|
+
return ordered
|
|
97
|
+
return sorted(random.Random(seed).sample(ordered, size), key=lambda r: r.id) # noqa: S311
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def run(
|
|
101
|
+
github: GitHubClient,
|
|
102
|
+
llm: LLMClient,
|
|
103
|
+
config: LLMConfig,
|
|
104
|
+
role: str,
|
|
105
|
+
template: PromptTemplate,
|
|
106
|
+
counter: TokenCounter,
|
|
107
|
+
reports: Sequence[dunlap.DunlapReport],
|
|
108
|
+
advisories: dict[str, AdvisoryText],
|
|
109
|
+
as_of: date,
|
|
110
|
+
sample_size: int | None,
|
|
111
|
+
seed: int = SEED,
|
|
112
|
+
ids: frozenset[str] | None = None,
|
|
113
|
+
split: str = "all",
|
|
114
|
+
split_sha256: str | None = None,
|
|
115
|
+
progress: Callable[[int, int], None] | None = None,
|
|
116
|
+
) -> ExtractionRun:
|
|
117
|
+
model = config.models[role]
|
|
118
|
+
budget = config.prompt_budget(role)
|
|
119
|
+
truth: dict[str, set[str]] = {}
|
|
120
|
+
predicted: dict[str, set[str]] = {}
|
|
121
|
+
naive: dict[str, set[str]] = {}
|
|
122
|
+
outcomes: list[ExtractionOutcome] = []
|
|
123
|
+
chosen = select(reports, sample_size, seed, ids)
|
|
124
|
+
for number, report in enumerate(chosen, 1):
|
|
125
|
+
mappings = [
|
|
126
|
+
map_commit(github, c.github_repo, c.sha, include=dunlap.paper_path_filter)
|
|
127
|
+
for c in report.commits
|
|
128
|
+
]
|
|
129
|
+
inputs, known = candidate_inputs(mappings)
|
|
130
|
+
outcome = extract(
|
|
131
|
+
llm,
|
|
132
|
+
model,
|
|
133
|
+
template,
|
|
134
|
+
advisories[report.id],
|
|
135
|
+
inputs,
|
|
136
|
+
known,
|
|
137
|
+
budget,
|
|
138
|
+
counter,
|
|
139
|
+
config.budget.max_hunk_lines,
|
|
140
|
+
)
|
|
141
|
+
outcomes.append(outcome)
|
|
142
|
+
truth[report.id] = set(report.symbols)
|
|
143
|
+
predicted[report.id] = set(outcome.vulnerable)
|
|
144
|
+
naive[report.id] = {i.name for i in inputs}
|
|
145
|
+
if progress is not None and (number % 10 == 0 or number == len(chosen)):
|
|
146
|
+
progress(number, len(chosen))
|
|
147
|
+
|
|
148
|
+
return ExtractionRun(
|
|
149
|
+
as_of=as_of,
|
|
150
|
+
model=model.model,
|
|
151
|
+
prompt_version=template.version,
|
|
152
|
+
reports=len(outcomes),
|
|
153
|
+
split=split,
|
|
154
|
+
split_sha256=split_sha256,
|
|
155
|
+
sample_size=sample_size,
|
|
156
|
+
sample_seed=seed,
|
|
157
|
+
extraction=score(predicted, truth, BOOTSTRAP, seed),
|
|
158
|
+
changedfuncs=score(naive, truth, BOOTSTRAP, seed),
|
|
159
|
+
status=dict(sorted(Counter(o.status for o in outcomes).items())),
|
|
160
|
+
truncated=sum(o.truncated for o in outcomes),
|
|
161
|
+
invented_ids_first_attempt=sum(o.invented_ids_first_attempt for o in outcomes),
|
|
162
|
+
entry_points_dropped=sum(o.entry_points_dropped for o in outcomes),
|
|
163
|
+
prompt_tokens=sum(o.prompt_tokens for o in outcomes),
|
|
164
|
+
completion_tokens=sum(o.completion_tokens for o in outcomes),
|
|
165
|
+
estimated_prompt_tokens=sum(o.estimated_prompt_tokens for o in outcomes),
|
|
166
|
+
calls=sum(o.calls for o in outcomes),
|
|
167
|
+
cached_calls=sum(o.cached_calls for o in outcomes),
|
|
168
|
+
provider_rejections=sum(o.provider_rejections for o in outcomes),
|
|
169
|
+
outcomes=tuple(outcomes),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _row(label: str, s: SetScores) -> str:
|
|
174
|
+
def ci(interval: Interval | None) -> str:
|
|
175
|
+
return "" if interval is None else f" ({interval.low:.2f} to {interval.high:.2f})"
|
|
176
|
+
|
|
177
|
+
return (
|
|
178
|
+
f"| {label} | {s.precision:.3f}{ci(s.precision_ci95)} | {s.recall:.3f}{ci(s.recall_ci95)} "
|
|
179
|
+
f"| {s.f1:.3f}{ci(s.f1_ci95)} | {s.miss_rate:.1%} |"
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def render_markdown(run_: ExtractionRun, source_lines: Sequence[str]) -> str:
|
|
184
|
+
split_note = f"split `{run_.split}`" + (
|
|
185
|
+
f" (SHA-256 `{run_.split_sha256}`)" if run_.split_sha256 else ""
|
|
186
|
+
)
|
|
187
|
+
if run_.sample_size is not None:
|
|
188
|
+
scope = (
|
|
189
|
+
f"a seeded random sample of {run_.reports} reports (seed {run_.sample_seed}) from "
|
|
190
|
+
f"{split_note}. **A sample run, not the benchmark result.**"
|
|
191
|
+
)
|
|
192
|
+
else:
|
|
193
|
+
scope = f"all {run_.reports} reports of {split_note}."
|
|
194
|
+
estimate_ratio = (
|
|
195
|
+
run_.prompt_tokens / run_.estimated_prompt_tokens if run_.estimated_prompt_tokens else 0.0
|
|
196
|
+
)
|
|
197
|
+
return "\n".join(
|
|
198
|
+
[
|
|
199
|
+
f"# LLM extraction on the Dunlap Go benchmark: `{run_.prompt_version}`, `{run_.model}`",
|
|
200
|
+
"",
|
|
201
|
+
"Generated by `python -m patchsight.pipeline dunlap-extract`; do not edit by hand.",
|
|
202
|
+
"",
|
|
203
|
+
"## Sources",
|
|
204
|
+
"",
|
|
205
|
+
*source_lines,
|
|
206
|
+
"",
|
|
207
|
+
"## Scope",
|
|
208
|
+
"",
|
|
209
|
+
f"Run on {scope}",
|
|
210
|
+
"",
|
|
211
|
+
"## Results",
|
|
212
|
+
"",
|
|
213
|
+
f"Unit: (report, function). 95% CIs from {BOOTSTRAP:,} bootstrap resamples of reports. "
|
|
214
|
+
"ChangedFuncs is scored on the same reports.",
|
|
215
|
+
"",
|
|
216
|
+
"| System | Precision | Recall | F1 | Miss rate |",
|
|
217
|
+
"|---|---|---|---|---|",
|
|
218
|
+
_row("ChangedFuncs (naive)", run_.changedfuncs),
|
|
219
|
+
_row(f"**{run_.model}**, `{run_.prompt_version}`", run_.extraction),
|
|
220
|
+
"",
|
|
221
|
+
"## Reliability",
|
|
222
|
+
"",
|
|
223
|
+
f"- Outcomes: {', '.join(f'{k} {v}' for k, v in run_.status.items())}. "
|
|
224
|
+
"A dead-lettered report predicts nothing in this table; the confidence tiers in "
|
|
225
|
+
"Phase 2 will fall back to ChangedFuncs instead.",
|
|
226
|
+
f"- Invented candidate IDs on the first attempt (pre-validation hallucination): "
|
|
227
|
+
f"{run_.invented_ids_first_attempt}. After validation: 0 by construction.",
|
|
228
|
+
f"- Entry-point names dropped because they matched no known definition: "
|
|
229
|
+
f"{run_.entry_points_dropped}.",
|
|
230
|
+
f"- Prompts truncated to the token budget: {run_.truncated}.",
|
|
231
|
+
"",
|
|
232
|
+
"## Cost",
|
|
233
|
+
"",
|
|
234
|
+
f"- Calls: {run_.calls} ({run_.cached_calls} from cache; "
|
|
235
|
+
f"{run_.provider_rejections} rejected by the provider and retried).",
|
|
236
|
+
f"- Tokens: {run_.prompt_tokens:,} prompt and {run_.completion_tokens:,} completion "
|
|
237
|
+
f"(mean {run_.prompt_tokens / max(run_.reports, 1):,.0f} prompt tokens per report).",
|
|
238
|
+
f"- Token count accuracy: the provider reported {estimate_ratio:.2f} times the tokens "
|
|
239
|
+
"PatchSight counted with the model's tokenizer.",
|
|
240
|
+
"",
|
|
241
|
+
]
|
|
242
|
+
)
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Replay bundles: re-score an LLM extraction run offline, from the model's own answers.
|
|
2
|
+
|
|
3
|
+
A run's prompts contain third-party code and cannot be published, but scoring needs only
|
|
4
|
+
what the model was shown *by ID* and what it answered. A replay bundle stores, per report:
|
|
5
|
+
|
|
6
|
+
* the rendered candidates: ID, function name, file path, and the hunk IDs each may cite;
|
|
7
|
+
* the model's raw answers, in order (``null`` where the provider rejected an attempt).
|
|
8
|
+
|
|
9
|
+
``replay`` runs the same validator and the same attempt logic as the live extractor on those
|
|
10
|
+
answers, so a stranger recomputes the published scores exactly, with no model, network or
|
|
11
|
+
code.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from collections.abc import Sequence
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from pydantic import BaseModel, ConfigDict
|
|
20
|
+
|
|
21
|
+
from patchsight.eval.metrics import SetScores, score
|
|
22
|
+
from patchsight.pipeline.benchmarks import dunlap
|
|
23
|
+
from patchsight.pipeline.benchmarks.extraction import candidate_inputs
|
|
24
|
+
from patchsight.pipeline.diffs.commits import map_commit
|
|
25
|
+
from patchsight.pipeline.diffs.github import GitHubClient
|
|
26
|
+
from patchsight.pipeline.extract.extractor import extract
|
|
27
|
+
from patchsight.pipeline.extract.prompt import (
|
|
28
|
+
AdvisoryText,
|
|
29
|
+
Candidate,
|
|
30
|
+
PromptTemplate,
|
|
31
|
+
RenderedPrompt,
|
|
32
|
+
render,
|
|
33
|
+
)
|
|
34
|
+
from patchsight.pipeline.extract.validate import ValidationFailedError, validate
|
|
35
|
+
from patchsight.pipeline.llm.client import (
|
|
36
|
+
LLMClient,
|
|
37
|
+
LLMRequest,
|
|
38
|
+
LLMResponse,
|
|
39
|
+
ProviderRejectedError,
|
|
40
|
+
RateLimiter,
|
|
41
|
+
)
|
|
42
|
+
from patchsight.pipeline.llm.config import LLMConfig
|
|
43
|
+
from patchsight.pipeline.llm.tokens import TokenCounter
|
|
44
|
+
|
|
45
|
+
REPLAY_VERSION = 1
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ReplayReport(BaseModel):
|
|
49
|
+
model_config = ConfigDict(frozen=True)
|
|
50
|
+
|
|
51
|
+
id: str
|
|
52
|
+
candidates: tuple[Candidate, ...]
|
|
53
|
+
answers: tuple[str | None, ...]
|
|
54
|
+
truncated: bool
|
|
55
|
+
dropped: tuple[str, ...]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ReplayBundle(BaseModel):
|
|
59
|
+
replay_version: int
|
|
60
|
+
benchmark: str
|
|
61
|
+
model: str
|
|
62
|
+
prompt_version: str
|
|
63
|
+
split: str = "all"
|
|
64
|
+
reports: tuple[ReplayReport, ...]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _CacheMissError(RuntimeError):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _cache_only(request: LLMRequest) -> tuple[str, int, int]:
|
|
72
|
+
raise _CacheMissError("a response is missing from the cache; export needs a complete run")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _Recorder(LLMClient):
|
|
76
|
+
"""A cache-only client that records every answer it serves, in order."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, cache_dir: Path, counter: TokenCounter) -> None:
|
|
79
|
+
super().__init__(_cache_only, RateLimiter(10**9, 10**9), cache_dir, counter)
|
|
80
|
+
self.answers: list[str | None] = []
|
|
81
|
+
|
|
82
|
+
def complete(self, request: LLMRequest) -> LLMResponse:
|
|
83
|
+
try:
|
|
84
|
+
response = super().complete(request)
|
|
85
|
+
except ProviderRejectedError:
|
|
86
|
+
self.answers.append(None)
|
|
87
|
+
raise
|
|
88
|
+
self.answers.append(response.content)
|
|
89
|
+
return response
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def export(
|
|
93
|
+
github: GitHubClient,
|
|
94
|
+
llm_cache: Path,
|
|
95
|
+
config: LLMConfig,
|
|
96
|
+
role: str,
|
|
97
|
+
template: PromptTemplate,
|
|
98
|
+
counter: TokenCounter,
|
|
99
|
+
reports: Sequence[dunlap.DunlapReport],
|
|
100
|
+
advisories: dict[str, AdvisoryText],
|
|
101
|
+
split: str = "all",
|
|
102
|
+
) -> ReplayBundle:
|
|
103
|
+
model = config.models[role]
|
|
104
|
+
budget = config.prompt_budget(role)
|
|
105
|
+
exported = []
|
|
106
|
+
for report in sorted(reports, key=lambda r: r.id):
|
|
107
|
+
mappings = [
|
|
108
|
+
map_commit(github, c.github_repo, c.sha, include=dunlap.paper_path_filter)
|
|
109
|
+
for c in report.commits
|
|
110
|
+
]
|
|
111
|
+
inputs, known = candidate_inputs(mappings)
|
|
112
|
+
prompt = render(
|
|
113
|
+
template, advisories[report.id], inputs, budget, counter, config.budget.max_hunk_lines
|
|
114
|
+
)
|
|
115
|
+
recorder = _Recorder(llm_cache, counter)
|
|
116
|
+
extract(
|
|
117
|
+
recorder, model, template, advisories[report.id], inputs, known, budget, counter,
|
|
118
|
+
config.budget.max_hunk_lines,
|
|
119
|
+
) # fmt: skip
|
|
120
|
+
exported.append(
|
|
121
|
+
ReplayReport(
|
|
122
|
+
id=report.id,
|
|
123
|
+
candidates=prompt.candidates,
|
|
124
|
+
answers=tuple(recorder.answers),
|
|
125
|
+
truncated=prompt.truncated,
|
|
126
|
+
dropped=prompt.dropped,
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
return ReplayBundle(
|
|
130
|
+
replay_version=REPLAY_VERSION,
|
|
131
|
+
benchmark="dunlap-dimva24",
|
|
132
|
+
model=model.model,
|
|
133
|
+
prompt_version=template.version,
|
|
134
|
+
split=split,
|
|
135
|
+
reports=tuple(exported),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def replay_report(report: ReplayReport, prompt_version: str) -> tuple[str, set[str]]:
|
|
140
|
+
"""(status, predicted vulnerable names), by the live extractor's own rules."""
|
|
141
|
+
if not report.candidates:
|
|
142
|
+
return "no_candidates", set()
|
|
143
|
+
prompt = RenderedPrompt(
|
|
144
|
+
version=prompt_version,
|
|
145
|
+
messages=(),
|
|
146
|
+
candidates=report.candidates,
|
|
147
|
+
hunk_ids=tuple(sorted({h for c in report.candidates for h in c.hunk_ids})),
|
|
148
|
+
estimated_tokens=0,
|
|
149
|
+
truncated=report.truncated,
|
|
150
|
+
dropped=report.dropped,
|
|
151
|
+
)
|
|
152
|
+
names = {c.id: c.name for c in report.candidates}
|
|
153
|
+
for attempt, answer in enumerate(report.answers[:2]):
|
|
154
|
+
if answer is None:
|
|
155
|
+
continue
|
|
156
|
+
try:
|
|
157
|
+
result = validate(answer, prompt, known_names=set())
|
|
158
|
+
except ValidationFailedError:
|
|
159
|
+
continue
|
|
160
|
+
clean_first = attempt == 0 and report.answers[0] is not None
|
|
161
|
+
return ("ok" if clean_first else "repaired"), {names[c] for c in result.vulnerable_ids}
|
|
162
|
+
return "dead_letter", set()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def replay(
|
|
166
|
+
bundle: ReplayBundle, truth: dict[str, set[str]], ids: frozenset[str] | None = None
|
|
167
|
+
) -> tuple[SetScores, dict[str, int]]:
|
|
168
|
+
chosen = [r for r in bundle.reports if ids is None or r.id in ids]
|
|
169
|
+
statuses: Counter[str] = Counter()
|
|
170
|
+
predicted: dict[str, set[str]] = {}
|
|
171
|
+
for report in chosen:
|
|
172
|
+
status, names = replay_report(report, bundle.prompt_version)
|
|
173
|
+
statuses[status] += 1
|
|
174
|
+
predicted[report.id] = names
|
|
175
|
+
return score(predicted, {r.id: truth[r.id] for r in chosen}), dict(sorted(statuses.items()))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def save(bundle: ReplayBundle, path: Path) -> None:
|
|
179
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
180
|
+
path.write_text(json.dumps(bundle.model_dump(mode="json"), indent=1) + "\n", encoding="utf-8")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def load(path: Path) -> ReplayBundle:
|
|
184
|
+
bundle = ReplayBundle.model_validate_json(path.read_text(encoding="utf-8"))
|
|
185
|
+
if bundle.replay_version != REPLAY_VERSION:
|
|
186
|
+
raise ValueError(f"{path} is replay version {bundle.replay_version}, not {REPLAY_VERSION}")
|
|
187
|
+
return bundle
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Frozen evaluation splits.
|
|
2
|
+
|
|
3
|
+
A split divides a benchmark's reports into ``dev`` (used for prompt and model choices) and
|
|
4
|
+
``test`` (reported, never tuned on). Reports that share a fix commit are grouped first and
|
|
5
|
+
always land in the same split; otherwise the same diff would sit on both sides.
|
|
6
|
+
|
|
7
|
+
A split is written once to ``splits/`` and committed. Its SHA-256 is recorded in every
|
|
8
|
+
report built from it, and the loader refuses a file whose content does not match its own
|
|
9
|
+
hash, so a split cannot change silently.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import random
|
|
15
|
+
from collections.abc import Iterable, Sequence
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict
|
|
19
|
+
|
|
20
|
+
from patchsight.pipeline.benchmarks.dunlap import DunlapReport
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SplitError(RuntimeError):
|
|
24
|
+
"""A split file is missing, tampered with, or would be overwritten."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Split(BaseModel):
|
|
28
|
+
model_config = ConfigDict(frozen=True)
|
|
29
|
+
|
|
30
|
+
name: str
|
|
31
|
+
benchmark: str
|
|
32
|
+
method: str
|
|
33
|
+
seed: int
|
|
34
|
+
dev: tuple[str, ...]
|
|
35
|
+
test: tuple[str, ...]
|
|
36
|
+
|
|
37
|
+
def ids(self, part: str) -> frozenset[str]:
|
|
38
|
+
if part == "dev":
|
|
39
|
+
return frozenset(self.dev)
|
|
40
|
+
if part == "test":
|
|
41
|
+
return frozenset(self.test)
|
|
42
|
+
if part == "all":
|
|
43
|
+
return frozenset(self.dev) | frozenset(self.test)
|
|
44
|
+
raise SplitError(f"unknown split part {part!r}; use dev, test or all")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class SplitFile(BaseModel):
|
|
48
|
+
split: Split
|
|
49
|
+
sha256: str
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _digest(split: Split) -> str:
|
|
53
|
+
canonical = json.dumps(split.model_dump(mode="json"), sort_keys=True)
|
|
54
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def commit_groups(reports: Iterable[DunlapReport]) -> list[tuple[str, ...]]:
|
|
58
|
+
"""Reports connected through any shared fix commit, each group sorted, groups sorted."""
|
|
59
|
+
parent: dict[str, str] = {}
|
|
60
|
+
|
|
61
|
+
def find(item: str) -> str:
|
|
62
|
+
parent.setdefault(item, item)
|
|
63
|
+
while parent[item] != item:
|
|
64
|
+
parent[item] = parent[parent[item]]
|
|
65
|
+
item = parent[item]
|
|
66
|
+
return item
|
|
67
|
+
|
|
68
|
+
first_report: dict[str, str] = {}
|
|
69
|
+
for report in reports:
|
|
70
|
+
find(report.id)
|
|
71
|
+
for commit in report.commits:
|
|
72
|
+
owner = first_report.setdefault(commit.sha, report.id)
|
|
73
|
+
a, b = find(owner), find(report.id)
|
|
74
|
+
if a != b:
|
|
75
|
+
parent[max(a, b)] = min(a, b)
|
|
76
|
+
|
|
77
|
+
groups: dict[str, list[str]] = {}
|
|
78
|
+
for report_id in parent:
|
|
79
|
+
groups.setdefault(find(report_id), []).append(report_id)
|
|
80
|
+
return sorted(tuple(sorted(g)) for g in groups.values())
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def make_split(reports: Sequence[DunlapReport], name: str, dev_size: int, seed: int) -> Split:
|
|
84
|
+
"""Shuffle commit groups with ``seed`` and fill ``dev`` until it holds ``dev_size``
|
|
85
|
+
reports (a group that would overshoot is skipped, so groups are never broken)."""
|
|
86
|
+
groups = commit_groups(reports)
|
|
87
|
+
random.Random(seed).shuffle(groups) # noqa: S311 - reproducible sampling, not security
|
|
88
|
+
dev: list[str] = []
|
|
89
|
+
for group in groups:
|
|
90
|
+
if len(dev) + len(group) <= dev_size:
|
|
91
|
+
dev.extend(group)
|
|
92
|
+
if len(dev) == dev_size:
|
|
93
|
+
break
|
|
94
|
+
dev_set = set(dev)
|
|
95
|
+
test = [r.id for r in reports if r.id not in dev_set]
|
|
96
|
+
return Split(
|
|
97
|
+
name=name,
|
|
98
|
+
benchmark="dunlap-dimva24",
|
|
99
|
+
method=f"seeded shuffle of commit-sharing groups; dev filled to {dev_size} reports",
|
|
100
|
+
seed=seed,
|
|
101
|
+
dev=tuple(sorted(dev)),
|
|
102
|
+
test=tuple(sorted(test)),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def save_split(split: Split, path: Path) -> SplitFile:
|
|
107
|
+
if path.exists():
|
|
108
|
+
raise SplitError(f"{path} already exists; frozen splits are never overwritten")
|
|
109
|
+
record = SplitFile(split=split, sha256=_digest(split))
|
|
110
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
path.write_text(record.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
|
112
|
+
return record
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def load_split(path: Path) -> SplitFile:
|
|
116
|
+
if not path.exists():
|
|
117
|
+
raise SplitError(f"{path} does not exist; create it with dunlap-split")
|
|
118
|
+
record = SplitFile.model_validate_json(path.read_text(encoding="utf-8"))
|
|
119
|
+
if _digest(record.split) != record.sha256:
|
|
120
|
+
raise SplitError(f"{path} does not match its recorded SHA-256; it was edited")
|
|
121
|
+
return record
|