patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,219 @@
1
+ """Render an extraction prompt: the advisory plus numbered candidates with their hunks.
2
+
3
+ * Candidates are numbered ``F1..Fn`` and hunks ``H1..Hm``. The model answers with these
4
+ IDs, so validation is a set lookup and invented names cannot get through.
5
+ * Untrusted text (the advisory and the code) goes inside tags. Any closing-tag sequence
6
+ inside it is neutralised, so data cannot break out of its tag.
7
+ * When the candidates exceed the token budget, they are ranked by BM25 against the
8
+ advisory text and kept in rank order until the budget is full. The result records
9
+ which candidates were dropped, so truncation is never silent.
10
+ """
11
+
12
+ import math
13
+ import re
14
+ from collections import Counter
15
+ from collections.abc import Sequence
16
+ from pathlib import Path
17
+
18
+ from pydantic import BaseModel, ConfigDict
19
+
20
+ from patchsight.pipeline.diffs.hunks import Hunk
21
+ from patchsight.pipeline.llm.tokens import MESSAGE_OVERHEAD, TokenCounter
22
+
23
+ _VERSION = re.compile(r"^version:\s*(\S+)\s*$", re.MULTILINE)
24
+ _SECTION = re.compile(r"^## (system|user)\s*$", re.MULTILINE)
25
+ _WORD = re.compile(r"[A-Za-z][A-Za-z0-9]*")
26
+ _CAMEL = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|\d+")
27
+
28
+
29
+ class PromptTemplate(BaseModel):
30
+ model_config = ConfigDict(frozen=True)
31
+
32
+ version: str
33
+ system: str
34
+ user: str
35
+
36
+
37
+ class AdvisoryText(BaseModel):
38
+ model_config = ConfigDict(frozen=True)
39
+
40
+ id: str
41
+ summary: str
42
+ details: str
43
+
44
+
45
+ class CandidateInput(BaseModel):
46
+ """A changed function and the hunks of its file that touch it."""
47
+
48
+ model_config = ConfigDict(frozen=True)
49
+
50
+ path: str
51
+ name: str
52
+ kind: str
53
+ hunks: tuple[Hunk, ...]
54
+ # Which commit the hunks come from, so two commits touching one file never collide.
55
+ source: str = ""
56
+
57
+
58
+ class Candidate(BaseModel):
59
+ model_config = ConfigDict(frozen=True)
60
+
61
+ id: str
62
+ path: str
63
+ name: str
64
+ hunk_ids: tuple[str, ...]
65
+
66
+
67
+ class RenderedPrompt(BaseModel):
68
+ model_config = ConfigDict(frozen=True)
69
+
70
+ version: str
71
+ messages: tuple[dict[str, str], ...]
72
+ candidates: tuple[Candidate, ...]
73
+ hunk_ids: tuple[str, ...]
74
+ estimated_tokens: int
75
+ truncated: bool
76
+ dropped: tuple[str, ...]
77
+
78
+
79
+ def load_template(path: Path) -> PromptTemplate:
80
+ text = path.read_text(encoding="utf-8")
81
+ version = _VERSION.search(text)
82
+ if version is None:
83
+ raise ValueError(f"{path} has no 'version:' line")
84
+ parts = _SECTION.split(text)
85
+ sections = {parts[i]: parts[i + 1].strip() for i in range(1, len(parts) - 1, 2)}
86
+ if set(sections) != {"system", "user"}:
87
+ raise ValueError(f"{path} needs exactly a '## system' and a '## user' section")
88
+ return PromptTemplate(
89
+ version=version.group(1), system=sections["system"], user=sections["user"]
90
+ )
91
+
92
+
93
+ def neutralise(text: str) -> str:
94
+ """Stop untrusted text from closing the tag it sits in, and drop control characters."""
95
+ cleaned = "".join(ch for ch in text if ch in "\n\t" or ch.isprintable())
96
+ return cleaned.replace("</", "<\\/")
97
+
98
+
99
+ def terms(text: str) -> list[str]:
100
+ """Lower-cased word pieces, splitting camelCase and snake_case."""
101
+ found: list[str] = []
102
+ for word in _WORD.findall(text):
103
+ found.extend(piece.lower() for piece in _CAMEL.findall(word) if len(piece) > 1)
104
+ return found
105
+
106
+
107
+ def bm25_rank(query: str, documents: Sequence[str], k1: float = 1.5, b: float = 0.75) -> list[int]:
108
+ """Document indices, best match first. Ties keep the original order."""
109
+ tokenised = [terms(doc) for doc in documents]
110
+ count = len(tokenised)
111
+ if not count:
112
+ return []
113
+ average = sum(len(t) for t in tokenised) / count or 1.0
114
+ frequency = Counter(term for t in tokenised for term in set(t))
115
+ query_terms = set(terms(query))
116
+ scores = []
117
+ for index, tokens in enumerate(tokenised):
118
+ counts = Counter(tokens)
119
+ score = 0.0
120
+ for term in query_terms:
121
+ tf = counts.get(term, 0)
122
+ if not tf:
123
+ continue
124
+ idf = math.log(1 + (count - frequency[term] + 0.5) / (frequency[term] + 0.5))
125
+ score += idf * tf * (k1 + 1) / (tf + k1 * (1 - b + b * len(tokens) / average))
126
+ scores.append((-score, index))
127
+ return [index for _, index in sorted(scores)]
128
+
129
+
130
+ def _hunk_block(hunk_id: str, hunk: Hunk, max_lines: int) -> str:
131
+ header = f"@@ -{hunk.old_start},{hunk.old_count} +{hunk.new_start},{hunk.new_count} @@"
132
+ lines = list(hunk.lines[:max_lines])
133
+ if len(hunk.lines) > max_lines:
134
+ lines.append(f"[... {len(hunk.lines) - max_lines} more lines not shown]")
135
+ body = neutralise("\n".join([header, *lines]))
136
+ return f'<hunk id="{hunk_id}">\n{body}\n</hunk>'
137
+
138
+
139
+ def render(
140
+ template: PromptTemplate,
141
+ advisory: AdvisoryText,
142
+ inputs: Sequence[CandidateInput],
143
+ budget_tokens: int,
144
+ counter: TokenCounter,
145
+ max_hunk_lines: int,
146
+ ) -> RenderedPrompt:
147
+ """Render the prompt, keeping as many candidates as fit within ``budget_tokens``."""
148
+
149
+ def estimate(text: str) -> int:
150
+ return counter.count(text)
151
+
152
+ advisory_text = neutralise(f"{advisory.summary}\n\n{advisory.details}".strip())
153
+
154
+ # Hunk IDs are global, in file and hunk order, so a hunk shared by two candidates has
155
+ # one ID that both can cite.
156
+ hunk_key = sorted({(c.source, c.path, h.index) for c in inputs for h in c.hunks})
157
+ hunk_id = {key: f"H{n}" for n, key in enumerate(hunk_key, 1)}
158
+
159
+ def block(cid: str, item: CandidateInput) -> str:
160
+ hunks = "\n".join(
161
+ _hunk_block(hunk_id[(item.source, item.path, h.index)], h, max_hunk_lines)
162
+ for h in sorted(item.hunks, key=lambda h: h.index)
163
+ )
164
+ attributes = f'id="{cid}" file="{neutralise(item.path)}" name="{neutralise(item.name)}"'
165
+ return f'<candidate {attributes} kind="{item.kind}">\n{hunks}\n</candidate>'
166
+
167
+ base = template.system + template.user.replace("{{advisory}}", advisory_text)
168
+ base_tokens = estimate(base) + 2 * MESSAGE_OVERHEAD
169
+ costs = [estimate(block("F0000", item)) for item in inputs]
170
+ order = bm25_rank(
171
+ advisory_text, [f"{i.name} {' '.join(h for x in i.hunks for h in x.lines)}" for i in inputs]
172
+ )
173
+
174
+ kept: set[int] = set()
175
+ used = base_tokens
176
+ for index in order:
177
+ # The best-ranked candidate is always kept; others only while they fit.
178
+ if kept and used + costs[index] > budget_tokens:
179
+ continue
180
+ kept.add(index)
181
+ used += costs[index]
182
+ kept_in_order = [i for i in range(len(inputs)) if i in kept]
183
+
184
+ candidates: list[Candidate] = []
185
+ blocks: list[str] = []
186
+ for number, index in enumerate(kept_in_order, 1):
187
+ item = inputs[index]
188
+ cid = f"F{number}"
189
+ candidates.append(
190
+ Candidate(
191
+ id=cid,
192
+ path=item.path,
193
+ name=item.name,
194
+ hunk_ids=tuple(
195
+ hunk_id[(item.source, item.path, h.index)]
196
+ for h in sorted(item.hunks, key=lambda h: h.index)
197
+ ),
198
+ )
199
+ )
200
+ blocks.append(block(cid, item))
201
+
202
+ user = (
203
+ template.user.replace("{{advisory_id}}", neutralise(advisory.id))
204
+ .replace("{{advisory}}", advisory_text)
205
+ .replace("{{candidates}}", "\n".join(blocks))
206
+ )
207
+ messages = ({"role": "system", "content": template.system}, {"role": "user", "content": user})
208
+ dropped = tuple(inputs[i].name for i in range(len(inputs)) if i not in kept)
209
+ return RenderedPrompt(
210
+ version=template.version,
211
+ messages=messages,
212
+ candidates=tuple(candidates),
213
+ hunk_ids=tuple(
214
+ sorted({h for c in candidates for h in c.hunk_ids}, key=lambda x: int(x[1:]))
215
+ ),
216
+ estimated_tokens=estimate(template.system) + estimate(user) + 2 * MESSAGE_OVERHEAD,
217
+ truncated=bool(dropped),
218
+ dropped=dropped,
219
+ )
@@ -0,0 +1,40 @@
1
+ """The contract between the LLM and the database: what the model must return."""
2
+
3
+ from typing import Any
4
+
5
+ from pydantic import BaseModel, ConfigDict
6
+
7
+ SCHEMA_NAME = "patchsight_extraction"
8
+
9
+
10
+ class ItemJudgement(BaseModel):
11
+ model_config = ConfigDict(extra="forbid")
12
+
13
+ candidate_id: str
14
+ is_vulnerable: bool
15
+ rationale: str
16
+ evidence_hunk_ids: list[str]
17
+
18
+
19
+ class Extraction(BaseModel):
20
+ model_config = ConfigDict(extra="forbid")
21
+
22
+ items: list[ItemJudgement]
23
+ public_entry_points: list[str]
24
+
25
+
26
+ def _inline(node: Any, definitions: dict[str, Any]) -> Any:
27
+ if isinstance(node, dict):
28
+ if "$ref" in node:
29
+ return _inline(definitions[node["$ref"].rsplit("/", 1)[-1]], definitions)
30
+ return {k: _inline(v, definitions) for k, v in node.items() if k not in ("$defs", "title")}
31
+ if isinstance(node, list):
32
+ return [_inline(item, definitions) for item in node]
33
+ return node
34
+
35
+
36
+ def response_schema() -> dict[str, Any]:
37
+ """The JSON schema sent to the provider, with references inlined for strict mode."""
38
+ raw = Extraction.model_json_schema()
39
+ result: dict[str, Any] = _inline(raw, raw.get("$defs", {}))
40
+ return result
@@ -0,0 +1,81 @@
1
+ """Deterministic checks on the model's answer. Nothing the model says is trusted unchecked.
2
+
3
+ * The JSON must match the schema.
4
+ * Every candidate gets exactly one judgement. Unknown IDs are counted as invented.
5
+ * Every vulnerable judgement cites at least one hunk, and every cited hunk must be shown
6
+ under that candidate.
7
+ * Entry points survive only if they name a definition known from the fetched source.
8
+ """
9
+
10
+ import json
11
+ from collections import Counter
12
+ from collections.abc import Set
13
+
14
+ from pydantic import BaseModel, ConfigDict, ValidationError
15
+
16
+ from patchsight.pipeline.extract.prompt import RenderedPrompt
17
+ from patchsight.pipeline.extract.schema import Extraction
18
+
19
+
20
+ class ValidationFailedError(ValueError):
21
+ def __init__(self, errors: list[str], invented_ids: int) -> None:
22
+ super().__init__("; ".join(errors))
23
+ self.errors = errors
24
+ self.invented_ids = invented_ids
25
+
26
+
27
+ class ValidatedExtraction(BaseModel):
28
+ model_config = ConfigDict(frozen=True)
29
+
30
+ vulnerable_ids: tuple[str, ...]
31
+ rationales: dict[str, str]
32
+ evidence: dict[str, tuple[str, ...]]
33
+ entry_points: tuple[str, ...]
34
+ entry_points_dropped: tuple[str, ...]
35
+
36
+
37
+ def validate(content: str, prompt: RenderedPrompt, known_names: Set[str]) -> ValidatedExtraction:
38
+ try:
39
+ extraction = Extraction.model_validate(json.loads(content))
40
+ except (json.JSONDecodeError, ValidationError) as error:
41
+ raise ValidationFailedError([f"not valid JSON for the schema: {error}"[:300]], 0) from None
42
+
43
+ owners = {c.id: set(c.hunk_ids) for c in prompt.candidates}
44
+ errors: list[str] = []
45
+ counts = Counter(item.candidate_id for item in extraction.items)
46
+ invented = sorted(i for i in counts if i not in owners)
47
+ if invented:
48
+ errors.append(f"unknown candidate IDs: {', '.join(invented)}")
49
+ duplicated = sorted(i for i, n in counts.items() if n > 1 and i in owners)
50
+ if duplicated:
51
+ errors.append(f"more than one item for: {', '.join(duplicated)}")
52
+ missing = sorted((set(owners) - set(counts)), key=lambda x: int(x[1:]))
53
+ if missing:
54
+ errors.append(f"no item for: {', '.join(missing)}")
55
+
56
+ for item in extraction.items:
57
+ if item.candidate_id not in owners:
58
+ continue
59
+ if not item.rationale.strip():
60
+ errors.append(f"{item.candidate_id}: empty rationale")
61
+ foreign = [h for h in item.evidence_hunk_ids if h not in owners[item.candidate_id]]
62
+ if foreign:
63
+ errors.append(
64
+ f"{item.candidate_id}: cites hunks not shown under it: {', '.join(foreign)}"
65
+ )
66
+ if item.is_vulnerable and not item.evidence_hunk_ids:
67
+ errors.append(f"{item.candidate_id}: vulnerable but cites no hunk")
68
+ if errors:
69
+ raise ValidationFailedError(errors, len(invented))
70
+
71
+ allowed = set(known_names) | {c.name for c in prompt.candidates}
72
+ kept = tuple(sorted({e for e in extraction.public_entry_points if e in allowed}))
73
+ dropped = tuple(sorted({e for e in extraction.public_entry_points if e not in allowed}))
74
+ judged = {item.candidate_id: item for item in extraction.items}
75
+ return ValidatedExtraction(
76
+ vulnerable_ids=tuple(c.id for c in prompt.candidates if judged[c.id].is_vulnerable),
77
+ rationales={cid: item.rationale for cid, item in judged.items()},
78
+ evidence={cid: tuple(item.evidence_hunk_ids) for cid, item in judged.items()},
79
+ entry_points=kept,
80
+ entry_points_dropped=dropped,
81
+ )