patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,242 @@
1
+ """Run LLM extraction on the Dunlap Go benchmark and score it against GoVulnDB symbols.
2
+
3
+ Candidates come from PatchSight's own fetch-and-map pipeline, the same candidate sets that
4
+ reproduce the published ChangedFuncs baseline. ChangedFuncs is scored on the same reports,
5
+ so every run carries its own naive baseline.
6
+ """
7
+
8
+ import csv
9
+ import random
10
+ from collections import Counter
11
+ from collections.abc import Callable, Sequence
12
+ from datetime import date
13
+ from pathlib import Path
14
+
15
+ from pydantic import BaseModel
16
+
17
+ from patchsight.eval.metrics import Interval, SetScores, score
18
+ from patchsight.pipeline.benchmarks import dunlap
19
+ from patchsight.pipeline.diffs.commits import CommitMapping, map_commit
20
+ from patchsight.pipeline.diffs.github import GitHubClient
21
+ from patchsight.pipeline.extract.extractor import ExtractionOutcome, extract
22
+ from patchsight.pipeline.extract.prompt import AdvisoryText, CandidateInput, PromptTemplate
23
+ from patchsight.pipeline.llm.client import LLMClient
24
+ from patchsight.pipeline.llm.config import LLMConfig
25
+ from patchsight.pipeline.llm.tokens import TokenCounter
26
+
27
+ DUNLAP_ADVISORIES = "data/advisory-data/govulndb-2023-10-26.csv"
28
+ BOOTSTRAP = 2000
29
+ SEED = 20261114
30
+
31
+
32
+ class ExtractionRun(BaseModel):
33
+ as_of: date
34
+ model: str
35
+ prompt_version: str
36
+ reports: int
37
+ split: str
38
+ split_sha256: str | None
39
+ sample_size: int | None
40
+ sample_seed: int
41
+ extraction: SetScores
42
+ changedfuncs: SetScores
43
+ status: dict[str, int]
44
+ truncated: int
45
+ invented_ids_first_attempt: int
46
+ entry_points_dropped: int
47
+ prompt_tokens: int
48
+ completion_tokens: int
49
+ estimated_prompt_tokens: int
50
+ calls: int
51
+ cached_calls: int
52
+ provider_rejections: int
53
+ outcomes: tuple[ExtractionOutcome, ...]
54
+
55
+
56
+ def load_advisories(path: Path) -> dict[str, AdvisoryText]:
57
+ csv.field_size_limit(1 << 30)
58
+ with path.open(newline="", encoding="utf-8") as handle:
59
+ return {
60
+ row["id"]: AdvisoryText(id=row["id"], summary=row["summary"], details=row["details"])
61
+ for row in csv.DictReader(handle)
62
+ }
63
+
64
+
65
+ def candidate_inputs(mappings: Sequence[CommitMapping]) -> tuple[list[CandidateInput], set[str]]:
66
+ """Function and method candidates from mapped commits, and every known definition name."""
67
+ inputs: list[CandidateInput] = []
68
+ known: set[str] = set()
69
+ for mapping in mappings:
70
+ for file in mapping.files:
71
+ known.update(file.mapping.known_names)
72
+ by_index = {h.index: h for h in file.hunks}
73
+ for changed in file.mapping.changed:
74
+ if changed.kind not in ("function", "method"):
75
+ continue
76
+ inputs.append(
77
+ CandidateInput(
78
+ path=file.mapping.path,
79
+ name=changed.name,
80
+ kind=changed.kind,
81
+ hunks=tuple(by_index[i] for i in changed.hunks if i in by_index),
82
+ source=mapping.sha,
83
+ )
84
+ )
85
+ return inputs, known
86
+
87
+
88
+ def select(
89
+ reports: Sequence[dunlap.DunlapReport],
90
+ size: int | None,
91
+ seed: int,
92
+ ids: frozenset[str] | None = None,
93
+ ) -> list[dunlap.DunlapReport]:
94
+ ordered = sorted((r for r in reports if ids is None or r.id in ids), key=lambda r: r.id)
95
+ if size is None or size >= len(ordered):
96
+ return ordered
97
+ return sorted(random.Random(seed).sample(ordered, size), key=lambda r: r.id) # noqa: S311
98
+
99
+
100
+ def run(
101
+ github: GitHubClient,
102
+ llm: LLMClient,
103
+ config: LLMConfig,
104
+ role: str,
105
+ template: PromptTemplate,
106
+ counter: TokenCounter,
107
+ reports: Sequence[dunlap.DunlapReport],
108
+ advisories: dict[str, AdvisoryText],
109
+ as_of: date,
110
+ sample_size: int | None,
111
+ seed: int = SEED,
112
+ ids: frozenset[str] | None = None,
113
+ split: str = "all",
114
+ split_sha256: str | None = None,
115
+ progress: Callable[[int, int], None] | None = None,
116
+ ) -> ExtractionRun:
117
+ model = config.models[role]
118
+ budget = config.prompt_budget(role)
119
+ truth: dict[str, set[str]] = {}
120
+ predicted: dict[str, set[str]] = {}
121
+ naive: dict[str, set[str]] = {}
122
+ outcomes: list[ExtractionOutcome] = []
123
+ chosen = select(reports, sample_size, seed, ids)
124
+ for number, report in enumerate(chosen, 1):
125
+ mappings = [
126
+ map_commit(github, c.github_repo, c.sha, include=dunlap.paper_path_filter)
127
+ for c in report.commits
128
+ ]
129
+ inputs, known = candidate_inputs(mappings)
130
+ outcome = extract(
131
+ llm,
132
+ model,
133
+ template,
134
+ advisories[report.id],
135
+ inputs,
136
+ known,
137
+ budget,
138
+ counter,
139
+ config.budget.max_hunk_lines,
140
+ )
141
+ outcomes.append(outcome)
142
+ truth[report.id] = set(report.symbols)
143
+ predicted[report.id] = set(outcome.vulnerable)
144
+ naive[report.id] = {i.name for i in inputs}
145
+ if progress is not None and (number % 10 == 0 or number == len(chosen)):
146
+ progress(number, len(chosen))
147
+
148
+ return ExtractionRun(
149
+ as_of=as_of,
150
+ model=model.model,
151
+ prompt_version=template.version,
152
+ reports=len(outcomes),
153
+ split=split,
154
+ split_sha256=split_sha256,
155
+ sample_size=sample_size,
156
+ sample_seed=seed,
157
+ extraction=score(predicted, truth, BOOTSTRAP, seed),
158
+ changedfuncs=score(naive, truth, BOOTSTRAP, seed),
159
+ status=dict(sorted(Counter(o.status for o in outcomes).items())),
160
+ truncated=sum(o.truncated for o in outcomes),
161
+ invented_ids_first_attempt=sum(o.invented_ids_first_attempt for o in outcomes),
162
+ entry_points_dropped=sum(o.entry_points_dropped for o in outcomes),
163
+ prompt_tokens=sum(o.prompt_tokens for o in outcomes),
164
+ completion_tokens=sum(o.completion_tokens for o in outcomes),
165
+ estimated_prompt_tokens=sum(o.estimated_prompt_tokens for o in outcomes),
166
+ calls=sum(o.calls for o in outcomes),
167
+ cached_calls=sum(o.cached_calls for o in outcomes),
168
+ provider_rejections=sum(o.provider_rejections for o in outcomes),
169
+ outcomes=tuple(outcomes),
170
+ )
171
+
172
+
173
+ def _row(label: str, s: SetScores) -> str:
174
+ def ci(interval: Interval | None) -> str:
175
+ return "" if interval is None else f" ({interval.low:.2f} to {interval.high:.2f})"
176
+
177
+ return (
178
+ f"| {label} | {s.precision:.3f}{ci(s.precision_ci95)} | {s.recall:.3f}{ci(s.recall_ci95)} "
179
+ f"| {s.f1:.3f}{ci(s.f1_ci95)} | {s.miss_rate:.1%} |"
180
+ )
181
+
182
+
183
+ def render_markdown(run_: ExtractionRun, source_lines: Sequence[str]) -> str:
184
+ split_note = f"split `{run_.split}`" + (
185
+ f" (SHA-256 `{run_.split_sha256}`)" if run_.split_sha256 else ""
186
+ )
187
+ if run_.sample_size is not None:
188
+ scope = (
189
+ f"a seeded random sample of {run_.reports} reports (seed {run_.sample_seed}) from "
190
+ f"{split_note}. **A sample run, not the benchmark result.**"
191
+ )
192
+ else:
193
+ scope = f"all {run_.reports} reports of {split_note}."
194
+ estimate_ratio = (
195
+ run_.prompt_tokens / run_.estimated_prompt_tokens if run_.estimated_prompt_tokens else 0.0
196
+ )
197
+ return "\n".join(
198
+ [
199
+ f"# LLM extraction on the Dunlap Go benchmark: `{run_.prompt_version}`, `{run_.model}`",
200
+ "",
201
+ "Generated by `python -m patchsight.pipeline dunlap-extract`; do not edit by hand.",
202
+ "",
203
+ "## Sources",
204
+ "",
205
+ *source_lines,
206
+ "",
207
+ "## Scope",
208
+ "",
209
+ f"Run on {scope}",
210
+ "",
211
+ "## Results",
212
+ "",
213
+ f"Unit: (report, function). 95% CIs from {BOOTSTRAP:,} bootstrap resamples of reports. "
214
+ "ChangedFuncs is scored on the same reports.",
215
+ "",
216
+ "| System | Precision | Recall | F1 | Miss rate |",
217
+ "|---|---|---|---|---|",
218
+ _row("ChangedFuncs (naive)", run_.changedfuncs),
219
+ _row(f"**{run_.model}**, `{run_.prompt_version}`", run_.extraction),
220
+ "",
221
+ "## Reliability",
222
+ "",
223
+ f"- Outcomes: {', '.join(f'{k} {v}' for k, v in run_.status.items())}. "
224
+ "A dead-lettered report predicts nothing in this table; the confidence tiers in "
225
+ "Phase 2 will fall back to ChangedFuncs instead.",
226
+ f"- Invented candidate IDs on the first attempt (pre-validation hallucination): "
227
+ f"{run_.invented_ids_first_attempt}. After validation: 0 by construction.",
228
+ f"- Entry-point names dropped because they matched no known definition: "
229
+ f"{run_.entry_points_dropped}.",
230
+ f"- Prompts truncated to the token budget: {run_.truncated}.",
231
+ "",
232
+ "## Cost",
233
+ "",
234
+ f"- Calls: {run_.calls} ({run_.cached_calls} from cache; "
235
+ f"{run_.provider_rejections} rejected by the provider and retried).",
236
+ f"- Tokens: {run_.prompt_tokens:,} prompt and {run_.completion_tokens:,} completion "
237
+ f"(mean {run_.prompt_tokens / max(run_.reports, 1):,.0f} prompt tokens per report).",
238
+ f"- Token count accuracy: the provider reported {estimate_ratio:.2f} times the tokens "
239
+ "PatchSight counted with the model's tokenizer.",
240
+ "",
241
+ ]
242
+ )
@@ -0,0 +1,187 @@
1
+ """Replay bundles: re-score an LLM extraction run offline, from the model's own answers.
2
+
3
+ A run's prompts contain third-party code and cannot be published, but scoring needs only
4
+ what the model was shown *by ID* and what it answered. A replay bundle stores, per report:
5
+
6
+ * the rendered candidates: ID, function name, file path, and the hunk IDs each may cite;
7
+ * the model's raw answers, in order (``null`` where the provider rejected an attempt).
8
+
9
+ ``replay`` runs the same validator and the same attempt logic as the live extractor on those
10
+ answers, so a stranger recomputes the published scores exactly, with no model, network or
11
+ code.
12
+ """
13
+
14
+ import json
15
+ from collections import Counter
16
+ from collections.abc import Sequence
17
+ from pathlib import Path
18
+
19
+ from pydantic import BaseModel, ConfigDict
20
+
21
+ from patchsight.eval.metrics import SetScores, score
22
+ from patchsight.pipeline.benchmarks import dunlap
23
+ from patchsight.pipeline.benchmarks.extraction import candidate_inputs
24
+ from patchsight.pipeline.diffs.commits import map_commit
25
+ from patchsight.pipeline.diffs.github import GitHubClient
26
+ from patchsight.pipeline.extract.extractor import extract
27
+ from patchsight.pipeline.extract.prompt import (
28
+ AdvisoryText,
29
+ Candidate,
30
+ PromptTemplate,
31
+ RenderedPrompt,
32
+ render,
33
+ )
34
+ from patchsight.pipeline.extract.validate import ValidationFailedError, validate
35
+ from patchsight.pipeline.llm.client import (
36
+ LLMClient,
37
+ LLMRequest,
38
+ LLMResponse,
39
+ ProviderRejectedError,
40
+ RateLimiter,
41
+ )
42
+ from patchsight.pipeline.llm.config import LLMConfig
43
+ from patchsight.pipeline.llm.tokens import TokenCounter
44
+
45
+ REPLAY_VERSION = 1
46
+
47
+
48
+ class ReplayReport(BaseModel):
49
+ model_config = ConfigDict(frozen=True)
50
+
51
+ id: str
52
+ candidates: tuple[Candidate, ...]
53
+ answers: tuple[str | None, ...]
54
+ truncated: bool
55
+ dropped: tuple[str, ...]
56
+
57
+
58
+ class ReplayBundle(BaseModel):
59
+ replay_version: int
60
+ benchmark: str
61
+ model: str
62
+ prompt_version: str
63
+ split: str = "all"
64
+ reports: tuple[ReplayReport, ...]
65
+
66
+
67
+ class _CacheMissError(RuntimeError):
68
+ pass
69
+
70
+
71
+ def _cache_only(request: LLMRequest) -> tuple[str, int, int]:
72
+ raise _CacheMissError("a response is missing from the cache; export needs a complete run")
73
+
74
+
75
+ class _Recorder(LLMClient):
76
+ """A cache-only client that records every answer it serves, in order."""
77
+
78
+ def __init__(self, cache_dir: Path, counter: TokenCounter) -> None:
79
+ super().__init__(_cache_only, RateLimiter(10**9, 10**9), cache_dir, counter)
80
+ self.answers: list[str | None] = []
81
+
82
+ def complete(self, request: LLMRequest) -> LLMResponse:
83
+ try:
84
+ response = super().complete(request)
85
+ except ProviderRejectedError:
86
+ self.answers.append(None)
87
+ raise
88
+ self.answers.append(response.content)
89
+ return response
90
+
91
+
92
+ def export(
93
+ github: GitHubClient,
94
+ llm_cache: Path,
95
+ config: LLMConfig,
96
+ role: str,
97
+ template: PromptTemplate,
98
+ counter: TokenCounter,
99
+ reports: Sequence[dunlap.DunlapReport],
100
+ advisories: dict[str, AdvisoryText],
101
+ split: str = "all",
102
+ ) -> ReplayBundle:
103
+ model = config.models[role]
104
+ budget = config.prompt_budget(role)
105
+ exported = []
106
+ for report in sorted(reports, key=lambda r: r.id):
107
+ mappings = [
108
+ map_commit(github, c.github_repo, c.sha, include=dunlap.paper_path_filter)
109
+ for c in report.commits
110
+ ]
111
+ inputs, known = candidate_inputs(mappings)
112
+ prompt = render(
113
+ template, advisories[report.id], inputs, budget, counter, config.budget.max_hunk_lines
114
+ )
115
+ recorder = _Recorder(llm_cache, counter)
116
+ extract(
117
+ recorder, model, template, advisories[report.id], inputs, known, budget, counter,
118
+ config.budget.max_hunk_lines,
119
+ ) # fmt: skip
120
+ exported.append(
121
+ ReplayReport(
122
+ id=report.id,
123
+ candidates=prompt.candidates,
124
+ answers=tuple(recorder.answers),
125
+ truncated=prompt.truncated,
126
+ dropped=prompt.dropped,
127
+ )
128
+ )
129
+ return ReplayBundle(
130
+ replay_version=REPLAY_VERSION,
131
+ benchmark="dunlap-dimva24",
132
+ model=model.model,
133
+ prompt_version=template.version,
134
+ split=split,
135
+ reports=tuple(exported),
136
+ )
137
+
138
+
139
+ def replay_report(report: ReplayReport, prompt_version: str) -> tuple[str, set[str]]:
140
+ """(status, predicted vulnerable names), by the live extractor's own rules."""
141
+ if not report.candidates:
142
+ return "no_candidates", set()
143
+ prompt = RenderedPrompt(
144
+ version=prompt_version,
145
+ messages=(),
146
+ candidates=report.candidates,
147
+ hunk_ids=tuple(sorted({h for c in report.candidates for h in c.hunk_ids})),
148
+ estimated_tokens=0,
149
+ truncated=report.truncated,
150
+ dropped=report.dropped,
151
+ )
152
+ names = {c.id: c.name for c in report.candidates}
153
+ for attempt, answer in enumerate(report.answers[:2]):
154
+ if answer is None:
155
+ continue
156
+ try:
157
+ result = validate(answer, prompt, known_names=set())
158
+ except ValidationFailedError:
159
+ continue
160
+ clean_first = attempt == 0 and report.answers[0] is not None
161
+ return ("ok" if clean_first else "repaired"), {names[c] for c in result.vulnerable_ids}
162
+ return "dead_letter", set()
163
+
164
+
165
+ def replay(
166
+ bundle: ReplayBundle, truth: dict[str, set[str]], ids: frozenset[str] | None = None
167
+ ) -> tuple[SetScores, dict[str, int]]:
168
+ chosen = [r for r in bundle.reports if ids is None or r.id in ids]
169
+ statuses: Counter[str] = Counter()
170
+ predicted: dict[str, set[str]] = {}
171
+ for report in chosen:
172
+ status, names = replay_report(report, bundle.prompt_version)
173
+ statuses[status] += 1
174
+ predicted[report.id] = names
175
+ return score(predicted, {r.id: truth[r.id] for r in chosen}), dict(sorted(statuses.items()))
176
+
177
+
178
+ def save(bundle: ReplayBundle, path: Path) -> None:
179
+ path.parent.mkdir(parents=True, exist_ok=True)
180
+ path.write_text(json.dumps(bundle.model_dump(mode="json"), indent=1) + "\n", encoding="utf-8")
181
+
182
+
183
+ def load(path: Path) -> ReplayBundle:
184
+ bundle = ReplayBundle.model_validate_json(path.read_text(encoding="utf-8"))
185
+ if bundle.replay_version != REPLAY_VERSION:
186
+ raise ValueError(f"{path} is replay version {bundle.replay_version}, not {REPLAY_VERSION}")
187
+ return bundle
@@ -0,0 +1,121 @@
1
+ """Frozen evaluation splits.
2
+
3
+ A split divides a benchmark's reports into ``dev`` (used for prompt and model choices) and
4
+ ``test`` (reported, never tuned on). Reports that share a fix commit are grouped first and
5
+ always land in the same split; otherwise the same diff would sit on both sides.
6
+
7
+ A split is written once to ``splits/`` and committed. Its SHA-256 is recorded in every
8
+ report built from it, and the loader refuses a file whose content does not match its own
9
+ hash, so a split cannot change silently.
10
+ """
11
+
12
+ import hashlib
13
+ import json
14
+ import random
15
+ from collections.abc import Iterable, Sequence
16
+ from pathlib import Path
17
+
18
+ from pydantic import BaseModel, ConfigDict
19
+
20
+ from patchsight.pipeline.benchmarks.dunlap import DunlapReport
21
+
22
+
23
+ class SplitError(RuntimeError):
24
+ """A split file is missing, tampered with, or would be overwritten."""
25
+
26
+
27
+ class Split(BaseModel):
28
+ model_config = ConfigDict(frozen=True)
29
+
30
+ name: str
31
+ benchmark: str
32
+ method: str
33
+ seed: int
34
+ dev: tuple[str, ...]
35
+ test: tuple[str, ...]
36
+
37
+ def ids(self, part: str) -> frozenset[str]:
38
+ if part == "dev":
39
+ return frozenset(self.dev)
40
+ if part == "test":
41
+ return frozenset(self.test)
42
+ if part == "all":
43
+ return frozenset(self.dev) | frozenset(self.test)
44
+ raise SplitError(f"unknown split part {part!r}; use dev, test or all")
45
+
46
+
47
+ class SplitFile(BaseModel):
48
+ split: Split
49
+ sha256: str
50
+
51
+
52
+ def _digest(split: Split) -> str:
53
+ canonical = json.dumps(split.model_dump(mode="json"), sort_keys=True)
54
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
55
+
56
+
57
+ def commit_groups(reports: Iterable[DunlapReport]) -> list[tuple[str, ...]]:
58
+ """Reports connected through any shared fix commit, each group sorted, groups sorted."""
59
+ parent: dict[str, str] = {}
60
+
61
+ def find(item: str) -> str:
62
+ parent.setdefault(item, item)
63
+ while parent[item] != item:
64
+ parent[item] = parent[parent[item]]
65
+ item = parent[item]
66
+ return item
67
+
68
+ first_report: dict[str, str] = {}
69
+ for report in reports:
70
+ find(report.id)
71
+ for commit in report.commits:
72
+ owner = first_report.setdefault(commit.sha, report.id)
73
+ a, b = find(owner), find(report.id)
74
+ if a != b:
75
+ parent[max(a, b)] = min(a, b)
76
+
77
+ groups: dict[str, list[str]] = {}
78
+ for report_id in parent:
79
+ groups.setdefault(find(report_id), []).append(report_id)
80
+ return sorted(tuple(sorted(g)) for g in groups.values())
81
+
82
+
83
+ def make_split(reports: Sequence[DunlapReport], name: str, dev_size: int, seed: int) -> Split:
84
+ """Shuffle commit groups with ``seed`` and fill ``dev`` until it holds ``dev_size``
85
+ reports (a group that would overshoot is skipped, so groups are never broken)."""
86
+ groups = commit_groups(reports)
87
+ random.Random(seed).shuffle(groups) # noqa: S311 - reproducible sampling, not security
88
+ dev: list[str] = []
89
+ for group in groups:
90
+ if len(dev) + len(group) <= dev_size:
91
+ dev.extend(group)
92
+ if len(dev) == dev_size:
93
+ break
94
+ dev_set = set(dev)
95
+ test = [r.id for r in reports if r.id not in dev_set]
96
+ return Split(
97
+ name=name,
98
+ benchmark="dunlap-dimva24",
99
+ method=f"seeded shuffle of commit-sharing groups; dev filled to {dev_size} reports",
100
+ seed=seed,
101
+ dev=tuple(sorted(dev)),
102
+ test=tuple(sorted(test)),
103
+ )
104
+
105
+
106
+ def save_split(split: Split, path: Path) -> SplitFile:
107
+ if path.exists():
108
+ raise SplitError(f"{path} already exists; frozen splits are never overwritten")
109
+ record = SplitFile(split=split, sha256=_digest(split))
110
+ path.parent.mkdir(parents=True, exist_ok=True)
111
+ path.write_text(record.model_dump_json(indent=2) + "\n", encoding="utf-8")
112
+ return record
113
+
114
+
115
+ def load_split(path: Path) -> SplitFile:
116
+ if not path.exists():
117
+ raise SplitError(f"{path} does not exist; create it with dunlap-split")
118
+ record = SplitFile.model_validate_json(path.read_text(encoding="utf-8"))
119
+ if _digest(record.split) != record.sha256:
120
+ raise SplitError(f"{path} does not match its recorded SHA-256; it was edited")
121
+ return record