patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Render an extraction prompt: the advisory plus numbered candidates with their hunks.
|
|
2
|
+
|
|
3
|
+
* Candidates are numbered ``F1..Fn`` and hunks ``H1..Hm``. The model answers with these
|
|
4
|
+
IDs, so validation is a set lookup and invented names cannot get through.
|
|
5
|
+
* Untrusted text (the advisory and the code) goes inside tags. Any closing-tag sequence
|
|
6
|
+
inside it is neutralised, so data cannot break out of its tag.
|
|
7
|
+
* When the candidates exceed the token budget, they are ranked by BM25 against the
|
|
8
|
+
advisory text and kept in rank order until the budget is full. The result records
|
|
9
|
+
which candidates were dropped, so truncation is never silent.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
import re
|
|
14
|
+
from collections import Counter
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict
|
|
19
|
+
|
|
20
|
+
from patchsight.pipeline.diffs.hunks import Hunk
|
|
21
|
+
from patchsight.pipeline.llm.tokens import MESSAGE_OVERHEAD, TokenCounter
|
|
22
|
+
|
|
23
|
+
_VERSION = re.compile(r"^version:\s*(\S+)\s*$", re.MULTILINE)
|
|
24
|
+
_SECTION = re.compile(r"^## (system|user)\s*$", re.MULTILINE)
|
|
25
|
+
_WORD = re.compile(r"[A-Za-z][A-Za-z0-9]*")
|
|
26
|
+
_CAMEL = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|\d+")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class PromptTemplate(BaseModel):
|
|
30
|
+
model_config = ConfigDict(frozen=True)
|
|
31
|
+
|
|
32
|
+
version: str
|
|
33
|
+
system: str
|
|
34
|
+
user: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class AdvisoryText(BaseModel):
|
|
38
|
+
model_config = ConfigDict(frozen=True)
|
|
39
|
+
|
|
40
|
+
id: str
|
|
41
|
+
summary: str
|
|
42
|
+
details: str
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class CandidateInput(BaseModel):
|
|
46
|
+
"""A changed function and the hunks of its file that touch it."""
|
|
47
|
+
|
|
48
|
+
model_config = ConfigDict(frozen=True)
|
|
49
|
+
|
|
50
|
+
path: str
|
|
51
|
+
name: str
|
|
52
|
+
kind: str
|
|
53
|
+
hunks: tuple[Hunk, ...]
|
|
54
|
+
# Which commit the hunks come from, so two commits touching one file never collide.
|
|
55
|
+
source: str = ""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Candidate(BaseModel):
|
|
59
|
+
model_config = ConfigDict(frozen=True)
|
|
60
|
+
|
|
61
|
+
id: str
|
|
62
|
+
path: str
|
|
63
|
+
name: str
|
|
64
|
+
hunk_ids: tuple[str, ...]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class RenderedPrompt(BaseModel):
|
|
68
|
+
model_config = ConfigDict(frozen=True)
|
|
69
|
+
|
|
70
|
+
version: str
|
|
71
|
+
messages: tuple[dict[str, str], ...]
|
|
72
|
+
candidates: tuple[Candidate, ...]
|
|
73
|
+
hunk_ids: tuple[str, ...]
|
|
74
|
+
estimated_tokens: int
|
|
75
|
+
truncated: bool
|
|
76
|
+
dropped: tuple[str, ...]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def load_template(path: Path) -> PromptTemplate:
|
|
80
|
+
text = path.read_text(encoding="utf-8")
|
|
81
|
+
version = _VERSION.search(text)
|
|
82
|
+
if version is None:
|
|
83
|
+
raise ValueError(f"{path} has no 'version:' line")
|
|
84
|
+
parts = _SECTION.split(text)
|
|
85
|
+
sections = {parts[i]: parts[i + 1].strip() for i in range(1, len(parts) - 1, 2)}
|
|
86
|
+
if set(sections) != {"system", "user"}:
|
|
87
|
+
raise ValueError(f"{path} needs exactly a '## system' and a '## user' section")
|
|
88
|
+
return PromptTemplate(
|
|
89
|
+
version=version.group(1), system=sections["system"], user=sections["user"]
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def neutralise(text: str) -> str:
|
|
94
|
+
"""Stop untrusted text from closing the tag it sits in, and drop control characters."""
|
|
95
|
+
cleaned = "".join(ch for ch in text if ch in "\n\t" or ch.isprintable())
|
|
96
|
+
return cleaned.replace("</", "<\\/")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def terms(text: str) -> list[str]:
|
|
100
|
+
"""Lower-cased word pieces, splitting camelCase and snake_case."""
|
|
101
|
+
found: list[str] = []
|
|
102
|
+
for word in _WORD.findall(text):
|
|
103
|
+
found.extend(piece.lower() for piece in _CAMEL.findall(word) if len(piece) > 1)
|
|
104
|
+
return found
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def bm25_rank(query: str, documents: Sequence[str], k1: float = 1.5, b: float = 0.75) -> list[int]:
|
|
108
|
+
"""Document indices, best match first. Ties keep the original order."""
|
|
109
|
+
tokenised = [terms(doc) for doc in documents]
|
|
110
|
+
count = len(tokenised)
|
|
111
|
+
if not count:
|
|
112
|
+
return []
|
|
113
|
+
average = sum(len(t) for t in tokenised) / count or 1.0
|
|
114
|
+
frequency = Counter(term for t in tokenised for term in set(t))
|
|
115
|
+
query_terms = set(terms(query))
|
|
116
|
+
scores = []
|
|
117
|
+
for index, tokens in enumerate(tokenised):
|
|
118
|
+
counts = Counter(tokens)
|
|
119
|
+
score = 0.0
|
|
120
|
+
for term in query_terms:
|
|
121
|
+
tf = counts.get(term, 0)
|
|
122
|
+
if not tf:
|
|
123
|
+
continue
|
|
124
|
+
idf = math.log(1 + (count - frequency[term] + 0.5) / (frequency[term] + 0.5))
|
|
125
|
+
score += idf * tf * (k1 + 1) / (tf + k1 * (1 - b + b * len(tokens) / average))
|
|
126
|
+
scores.append((-score, index))
|
|
127
|
+
return [index for _, index in sorted(scores)]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _hunk_block(hunk_id: str, hunk: Hunk, max_lines: int) -> str:
|
|
131
|
+
header = f"@@ -{hunk.old_start},{hunk.old_count} +{hunk.new_start},{hunk.new_count} @@"
|
|
132
|
+
lines = list(hunk.lines[:max_lines])
|
|
133
|
+
if len(hunk.lines) > max_lines:
|
|
134
|
+
lines.append(f"[... {len(hunk.lines) - max_lines} more lines not shown]")
|
|
135
|
+
body = neutralise("\n".join([header, *lines]))
|
|
136
|
+
return f'<hunk id="{hunk_id}">\n{body}\n</hunk>'
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def render(
|
|
140
|
+
template: PromptTemplate,
|
|
141
|
+
advisory: AdvisoryText,
|
|
142
|
+
inputs: Sequence[CandidateInput],
|
|
143
|
+
budget_tokens: int,
|
|
144
|
+
counter: TokenCounter,
|
|
145
|
+
max_hunk_lines: int,
|
|
146
|
+
) -> RenderedPrompt:
|
|
147
|
+
"""Render the prompt, keeping as many candidates as fit within ``budget_tokens``."""
|
|
148
|
+
|
|
149
|
+
def estimate(text: str) -> int:
|
|
150
|
+
return counter.count(text)
|
|
151
|
+
|
|
152
|
+
advisory_text = neutralise(f"{advisory.summary}\n\n{advisory.details}".strip())
|
|
153
|
+
|
|
154
|
+
# Hunk IDs are global, in file and hunk order, so a hunk shared by two candidates has
|
|
155
|
+
# one ID that both can cite.
|
|
156
|
+
hunk_key = sorted({(c.source, c.path, h.index) for c in inputs for h in c.hunks})
|
|
157
|
+
hunk_id = {key: f"H{n}" for n, key in enumerate(hunk_key, 1)}
|
|
158
|
+
|
|
159
|
+
def block(cid: str, item: CandidateInput) -> str:
|
|
160
|
+
hunks = "\n".join(
|
|
161
|
+
_hunk_block(hunk_id[(item.source, item.path, h.index)], h, max_hunk_lines)
|
|
162
|
+
for h in sorted(item.hunks, key=lambda h: h.index)
|
|
163
|
+
)
|
|
164
|
+
attributes = f'id="{cid}" file="{neutralise(item.path)}" name="{neutralise(item.name)}"'
|
|
165
|
+
return f'<candidate {attributes} kind="{item.kind}">\n{hunks}\n</candidate>'
|
|
166
|
+
|
|
167
|
+
base = template.system + template.user.replace("{{advisory}}", advisory_text)
|
|
168
|
+
base_tokens = estimate(base) + 2 * MESSAGE_OVERHEAD
|
|
169
|
+
costs = [estimate(block("F0000", item)) for item in inputs]
|
|
170
|
+
order = bm25_rank(
|
|
171
|
+
advisory_text, [f"{i.name} {' '.join(h for x in i.hunks for h in x.lines)}" for i in inputs]
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
kept: set[int] = set()
|
|
175
|
+
used = base_tokens
|
|
176
|
+
for index in order:
|
|
177
|
+
# The best-ranked candidate is always kept; others only while they fit.
|
|
178
|
+
if kept and used + costs[index] > budget_tokens:
|
|
179
|
+
continue
|
|
180
|
+
kept.add(index)
|
|
181
|
+
used += costs[index]
|
|
182
|
+
kept_in_order = [i for i in range(len(inputs)) if i in kept]
|
|
183
|
+
|
|
184
|
+
candidates: list[Candidate] = []
|
|
185
|
+
blocks: list[str] = []
|
|
186
|
+
for number, index in enumerate(kept_in_order, 1):
|
|
187
|
+
item = inputs[index]
|
|
188
|
+
cid = f"F{number}"
|
|
189
|
+
candidates.append(
|
|
190
|
+
Candidate(
|
|
191
|
+
id=cid,
|
|
192
|
+
path=item.path,
|
|
193
|
+
name=item.name,
|
|
194
|
+
hunk_ids=tuple(
|
|
195
|
+
hunk_id[(item.source, item.path, h.index)]
|
|
196
|
+
for h in sorted(item.hunks, key=lambda h: h.index)
|
|
197
|
+
),
|
|
198
|
+
)
|
|
199
|
+
)
|
|
200
|
+
blocks.append(block(cid, item))
|
|
201
|
+
|
|
202
|
+
user = (
|
|
203
|
+
template.user.replace("{{advisory_id}}", neutralise(advisory.id))
|
|
204
|
+
.replace("{{advisory}}", advisory_text)
|
|
205
|
+
.replace("{{candidates}}", "\n".join(blocks))
|
|
206
|
+
)
|
|
207
|
+
messages = ({"role": "system", "content": template.system}, {"role": "user", "content": user})
|
|
208
|
+
dropped = tuple(inputs[i].name for i in range(len(inputs)) if i not in kept)
|
|
209
|
+
return RenderedPrompt(
|
|
210
|
+
version=template.version,
|
|
211
|
+
messages=messages,
|
|
212
|
+
candidates=tuple(candidates),
|
|
213
|
+
hunk_ids=tuple(
|
|
214
|
+
sorted({h for c in candidates for h in c.hunk_ids}, key=lambda x: int(x[1:]))
|
|
215
|
+
),
|
|
216
|
+
estimated_tokens=estimate(template.system) + estimate(user) + 2 * MESSAGE_OVERHEAD,
|
|
217
|
+
truncated=bool(dropped),
|
|
218
|
+
dropped=dropped,
|
|
219
|
+
)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""The contract between the LLM and the database: what the model must return."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict
|
|
6
|
+
|
|
7
|
+
SCHEMA_NAME = "patchsight_extraction"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ItemJudgement(BaseModel):
|
|
11
|
+
model_config = ConfigDict(extra="forbid")
|
|
12
|
+
|
|
13
|
+
candidate_id: str
|
|
14
|
+
is_vulnerable: bool
|
|
15
|
+
rationale: str
|
|
16
|
+
evidence_hunk_ids: list[str]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Extraction(BaseModel):
|
|
20
|
+
model_config = ConfigDict(extra="forbid")
|
|
21
|
+
|
|
22
|
+
items: list[ItemJudgement]
|
|
23
|
+
public_entry_points: list[str]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _inline(node: Any, definitions: dict[str, Any]) -> Any:
|
|
27
|
+
if isinstance(node, dict):
|
|
28
|
+
if "$ref" in node:
|
|
29
|
+
return _inline(definitions[node["$ref"].rsplit("/", 1)[-1]], definitions)
|
|
30
|
+
return {k: _inline(v, definitions) for k, v in node.items() if k not in ("$defs", "title")}
|
|
31
|
+
if isinstance(node, list):
|
|
32
|
+
return [_inline(item, definitions) for item in node]
|
|
33
|
+
return node
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def response_schema() -> dict[str, Any]:
|
|
37
|
+
"""The JSON schema sent to the provider, with references inlined for strict mode."""
|
|
38
|
+
raw = Extraction.model_json_schema()
|
|
39
|
+
result: dict[str, Any] = _inline(raw, raw.get("$defs", {}))
|
|
40
|
+
return result
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Deterministic checks on the model's answer. Nothing the model says is trusted unchecked.
|
|
2
|
+
|
|
3
|
+
* The JSON must match the schema.
|
|
4
|
+
* Every candidate gets exactly one judgement. Unknown IDs are counted as invented.
|
|
5
|
+
* Every vulnerable judgement cites at least one hunk, and every cited hunk must be shown
|
|
6
|
+
under that candidate.
|
|
7
|
+
* Entry points survive only if they name a definition known from the fetched source.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from collections.abc import Set
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, ValidationError
|
|
15
|
+
|
|
16
|
+
from patchsight.pipeline.extract.prompt import RenderedPrompt
|
|
17
|
+
from patchsight.pipeline.extract.schema import Extraction
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ValidationFailedError(ValueError):
|
|
21
|
+
def __init__(self, errors: list[str], invented_ids: int) -> None:
|
|
22
|
+
super().__init__("; ".join(errors))
|
|
23
|
+
self.errors = errors
|
|
24
|
+
self.invented_ids = invented_ids
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ValidatedExtraction(BaseModel):
|
|
28
|
+
model_config = ConfigDict(frozen=True)
|
|
29
|
+
|
|
30
|
+
vulnerable_ids: tuple[str, ...]
|
|
31
|
+
rationales: dict[str, str]
|
|
32
|
+
evidence: dict[str, tuple[str, ...]]
|
|
33
|
+
entry_points: tuple[str, ...]
|
|
34
|
+
entry_points_dropped: tuple[str, ...]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def validate(content: str, prompt: RenderedPrompt, known_names: Set[str]) -> ValidatedExtraction:
|
|
38
|
+
try:
|
|
39
|
+
extraction = Extraction.model_validate(json.loads(content))
|
|
40
|
+
except (json.JSONDecodeError, ValidationError) as error:
|
|
41
|
+
raise ValidationFailedError([f"not valid JSON for the schema: {error}"[:300]], 0) from None
|
|
42
|
+
|
|
43
|
+
owners = {c.id: set(c.hunk_ids) for c in prompt.candidates}
|
|
44
|
+
errors: list[str] = []
|
|
45
|
+
counts = Counter(item.candidate_id for item in extraction.items)
|
|
46
|
+
invented = sorted(i for i in counts if i not in owners)
|
|
47
|
+
if invented:
|
|
48
|
+
errors.append(f"unknown candidate IDs: {', '.join(invented)}")
|
|
49
|
+
duplicated = sorted(i for i, n in counts.items() if n > 1 and i in owners)
|
|
50
|
+
if duplicated:
|
|
51
|
+
errors.append(f"more than one item for: {', '.join(duplicated)}")
|
|
52
|
+
missing = sorted((set(owners) - set(counts)), key=lambda x: int(x[1:]))
|
|
53
|
+
if missing:
|
|
54
|
+
errors.append(f"no item for: {', '.join(missing)}")
|
|
55
|
+
|
|
56
|
+
for item in extraction.items:
|
|
57
|
+
if item.candidate_id not in owners:
|
|
58
|
+
continue
|
|
59
|
+
if not item.rationale.strip():
|
|
60
|
+
errors.append(f"{item.candidate_id}: empty rationale")
|
|
61
|
+
foreign = [h for h in item.evidence_hunk_ids if h not in owners[item.candidate_id]]
|
|
62
|
+
if foreign:
|
|
63
|
+
errors.append(
|
|
64
|
+
f"{item.candidate_id}: cites hunks not shown under it: {', '.join(foreign)}"
|
|
65
|
+
)
|
|
66
|
+
if item.is_vulnerable and not item.evidence_hunk_ids:
|
|
67
|
+
errors.append(f"{item.candidate_id}: vulnerable but cites no hunk")
|
|
68
|
+
if errors:
|
|
69
|
+
raise ValidationFailedError(errors, len(invented))
|
|
70
|
+
|
|
71
|
+
allowed = set(known_names) | {c.name for c in prompt.candidates}
|
|
72
|
+
kept = tuple(sorted({e for e in extraction.public_entry_points if e in allowed}))
|
|
73
|
+
dropped = tuple(sorted({e for e in extraction.public_entry_points if e not in allowed}))
|
|
74
|
+
judged = {item.candidate_id: item for item in extraction.items}
|
|
75
|
+
return ValidatedExtraction(
|
|
76
|
+
vulnerable_ids=tuple(c.id for c in prompt.candidates if judged[c.id].is_vulnerable),
|
|
77
|
+
rationales={cid: item.rationale for cid, item in judged.items()},
|
|
78
|
+
evidence={cid: tuple(item.evidence_hunk_ids) for cid, item in judged.items()},
|
|
79
|
+
entry_points=kept,
|
|
80
|
+
entry_points_dropped=dropped,
|
|
81
|
+
)
|