patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Map diff hunks to the functions, methods and classes they change, using tree-sitter.
|
|
2
|
+
|
|
3
|
+
This is the paper's ChangedFuncs step (Dunlap et al., Algorithm 1), for Python and Go:
|
|
4
|
+
|
|
5
|
+
* Every hunk is mapped on both sides. Removed lines are looked up in the pre-fix file and
|
|
6
|
+
added lines in the post-fix file. A definition changed on either side counts once.
|
|
7
|
+
* A change belongs to the innermost *addressable* definition:
|
|
8
|
+
* Python: a module-level function, a class, or a method, including methods of nested
|
|
9
|
+
classes (``Outer.Inner.method``). Nested functions belong to their enclosing function,
|
|
10
|
+
because user code cannot reference them directly. A change in a class body outside any
|
|
11
|
+
method belongs to the class.
|
|
12
|
+
* Go: a function (``Parse``) or a method, named ``Type.Method`` with pointers and type
|
|
13
|
+
parameters removed. This matches the Go vulnerability database's symbol format.
|
|
14
|
+
Function literals belong to their enclosing function.
|
|
15
|
+
* Changed lines outside any definition (imports, module constants, Go type
|
|
16
|
+
declarations) are recorded as module-level changes. They are not function candidates.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from collections.abc import Iterable, Iterator, Sequence
|
|
20
|
+
from functools import cache
|
|
21
|
+
from typing import Literal
|
|
22
|
+
|
|
23
|
+
import tree_sitter_go
|
|
24
|
+
import tree_sitter_python
|
|
25
|
+
from pydantic import BaseModel, ConfigDict
|
|
26
|
+
from tree_sitter import Language, Node, Parser
|
|
27
|
+
|
|
28
|
+
from patchsight.pipeline.diffs.hunks import Hunk
|
|
29
|
+
|
|
30
|
+
SourceLanguage = Literal["python", "go"]
|
|
31
|
+
DefinitionKind = Literal["function", "method", "class"]
|
|
32
|
+
Side = Literal["pre", "post"]
|
|
33
|
+
|
|
34
|
+
_LANGUAGE_BY_SUFFIX: dict[str, SourceLanguage] = {".py": "python", ".pyi": "python", ".go": "go"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Definition(BaseModel):
|
|
38
|
+
"""A named definition with its 1-based, inclusive line range."""
|
|
39
|
+
|
|
40
|
+
model_config = ConfigDict(frozen=True)
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
kind: DefinitionKind
|
|
44
|
+
start_line: int
|
|
45
|
+
end_line: int
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ChangedDefinition(BaseModel):
|
|
49
|
+
model_config = ConfigDict(frozen=True)
|
|
50
|
+
|
|
51
|
+
path: str
|
|
52
|
+
name: str
|
|
53
|
+
kind: DefinitionKind
|
|
54
|
+
sides: tuple[Side, ...]
|
|
55
|
+
hunks: tuple[int, ...]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class FileMapping(BaseModel):
|
|
59
|
+
model_config = ConfigDict(frozen=True)
|
|
60
|
+
|
|
61
|
+
path: str
|
|
62
|
+
changed: tuple[ChangedDefinition, ...]
|
|
63
|
+
module_level_hunks: tuple[int, ...]
|
|
64
|
+
parse_errors: bool
|
|
65
|
+
# Every definition name in either version of the file, for validating names.
|
|
66
|
+
known_names: tuple[str, ...] = ()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def language_for(path: str) -> SourceLanguage | None:
|
|
70
|
+
for suffix, language in _LANGUAGE_BY_SUFFIX.items():
|
|
71
|
+
if path.endswith(suffix):
|
|
72
|
+
return language
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@cache
|
|
77
|
+
def _parser(language: SourceLanguage) -> Parser:
|
|
78
|
+
grammar = tree_sitter_python if language == "python" else tree_sitter_go
|
|
79
|
+
return Parser(Language(grammar.language()))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _text(node: Node | None) -> str:
|
|
83
|
+
return node.text.decode("utf-8", "replace") if node is not None and node.text else ""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _lines(node: Node) -> tuple[int, int]:
|
|
87
|
+
return node.start_point.row + 1, node.end_point.row + 1
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _python_definitions(node: Node, scope: tuple[str, ...], in_class: bool) -> Iterator[Definition]:
|
|
91
|
+
for child in node.children:
|
|
92
|
+
target, outer = child, child
|
|
93
|
+
if child.type == "decorated_definition":
|
|
94
|
+
target = child.child_by_field_name("definition") or child
|
|
95
|
+
if target.type not in ("function_definition", "class_definition"):
|
|
96
|
+
continue
|
|
97
|
+
name = _text(target.child_by_field_name("name"))
|
|
98
|
+
if not name:
|
|
99
|
+
continue
|
|
100
|
+
start, end = _lines(outer) # decorators belong to the definition
|
|
101
|
+
qualified = ".".join((*scope, name))
|
|
102
|
+
if target.type == "class_definition":
|
|
103
|
+
yield Definition(name=qualified, kind="class", start_line=start, end_line=end)
|
|
104
|
+
body = target.child_by_field_name("body")
|
|
105
|
+
if body is not None:
|
|
106
|
+
yield from _python_definitions(body, (*scope, name), in_class=True)
|
|
107
|
+
else:
|
|
108
|
+
kind: DefinitionKind = "method" if in_class else "function"
|
|
109
|
+
yield Definition(name=qualified, kind=kind, start_line=start, end_line=end)
|
|
110
|
+
# Nested functions are part of this one: do not descend.
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _go_receiver_type(receiver: Node | None) -> str:
|
|
114
|
+
if receiver is None:
|
|
115
|
+
return ""
|
|
116
|
+
stack = [receiver]
|
|
117
|
+
while stack:
|
|
118
|
+
current = stack.pop(0)
|
|
119
|
+
if current.type == "type_identifier":
|
|
120
|
+
return _text(current)
|
|
121
|
+
stack.extend(current.children)
|
|
122
|
+
return ""
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _go_definitions(root: Node) -> Iterator[Definition]:
|
|
126
|
+
for child in root.children:
|
|
127
|
+
if child.type == "function_declaration":
|
|
128
|
+
name = _text(child.child_by_field_name("name"))
|
|
129
|
+
kind: DefinitionKind = "function"
|
|
130
|
+
elif child.type == "method_declaration":
|
|
131
|
+
receiver = _go_receiver_type(child.child_by_field_name("receiver"))
|
|
132
|
+
method = _text(child.child_by_field_name("name"))
|
|
133
|
+
name = f"{receiver}.{method}" if receiver else method
|
|
134
|
+
kind = "method"
|
|
135
|
+
else:
|
|
136
|
+
continue
|
|
137
|
+
if name:
|
|
138
|
+
start, end = _lines(child)
|
|
139
|
+
yield Definition(name=name, kind=kind, start_line=start, end_line=end)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def definitions(source: bytes, language: SourceLanguage) -> tuple[list[Definition], bool]:
|
|
143
|
+
"""All addressable definitions in ``source``, and whether the parse had errors."""
|
|
144
|
+
tree = _parser(language).parse(source)
|
|
145
|
+
root = tree.root_node
|
|
146
|
+
found = (
|
|
147
|
+
list(_python_definitions(root, (), in_class=False))
|
|
148
|
+
if language == "python"
|
|
149
|
+
else list(_go_definitions(root))
|
|
150
|
+
)
|
|
151
|
+
return found, root.has_error
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _innermost(defs: Sequence[Definition], line: int) -> Definition | None:
|
|
155
|
+
containing = [d for d in defs if d.start_line <= line <= d.end_line]
|
|
156
|
+
if not containing:
|
|
157
|
+
return None
|
|
158
|
+
return min(containing, key=lambda d: (d.end_line - d.start_line, -d.start_line))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def map_hunks(
|
|
162
|
+
path: str,
|
|
163
|
+
language: SourceLanguage,
|
|
164
|
+
old_source: bytes | None,
|
|
165
|
+
new_source: bytes | None,
|
|
166
|
+
hunks: Iterable[Hunk],
|
|
167
|
+
) -> FileMapping:
|
|
168
|
+
"""Map each hunk's changed lines to the definitions they fall in."""
|
|
169
|
+
old_defs, old_errors = definitions(old_source, language) if old_source else ([], False)
|
|
170
|
+
new_defs, new_errors = definitions(new_source, language) if new_source else ([], False)
|
|
171
|
+
changed: dict[str, tuple[DefinitionKind, set[Side], set[int]]] = {}
|
|
172
|
+
module_level: set[int] = set()
|
|
173
|
+
|
|
174
|
+
def note(definition: Definition | None, side: Side, hunk: int) -> None:
|
|
175
|
+
if definition is None:
|
|
176
|
+
module_level.add(hunk)
|
|
177
|
+
return
|
|
178
|
+
_, sides, hunk_ids = changed.setdefault(definition.name, (definition.kind, set(), set()))
|
|
179
|
+
sides.add(side)
|
|
180
|
+
hunk_ids.add(hunk)
|
|
181
|
+
|
|
182
|
+
for hunk in hunks:
|
|
183
|
+
for line in hunk.removed:
|
|
184
|
+
note(_innermost(old_defs, line), "pre", hunk.index)
|
|
185
|
+
for line in hunk.added:
|
|
186
|
+
note(_innermost(new_defs, line), "post", hunk.index)
|
|
187
|
+
|
|
188
|
+
return FileMapping(
|
|
189
|
+
path=path,
|
|
190
|
+
changed=tuple(
|
|
191
|
+
ChangedDefinition(
|
|
192
|
+
path=path,
|
|
193
|
+
name=name,
|
|
194
|
+
kind=kind,
|
|
195
|
+
sides=tuple(s for s in ("pre", "post") if s in sides),
|
|
196
|
+
hunks=tuple(sorted(hunk_ids)),
|
|
197
|
+
)
|
|
198
|
+
for name, (kind, sides, hunk_ids) in sorted(changed.items())
|
|
199
|
+
),
|
|
200
|
+
module_level_hunks=tuple(sorted(module_level)),
|
|
201
|
+
parse_errors=old_errors or new_errors,
|
|
202
|
+
known_names=tuple(sorted({d.name for d in (*old_defs, *new_defs)})),
|
|
203
|
+
)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Classify changed file paths, so tests, docs, changelogs and vendored code are dropped.
|
|
2
|
+
|
|
3
|
+
Proposal §4, preprocessing step 3. Only source kinds go on to hunk mapping; ``python`` is
|
|
4
|
+
the kind tree-sitter maps to functions. Rules are checked in a fixed order, so a test file
|
|
5
|
+
inside a vendored directory counts as vendored, and so on.
|
|
6
|
+
|
|
7
|
+
Dropping real source loses recall, which is the dangerous direction, so rules lean towards
|
|
8
|
+
keeping code. Known limitation: a package that ships a ``test/`` module as public API (for
|
|
9
|
+
example ``django/test/``) is classified as tests; such misses are tagged in failure analysis.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from pathlib import PurePosixPath
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
PathKind = Literal[
|
|
16
|
+
"python",
|
|
17
|
+
"native",
|
|
18
|
+
"other_source",
|
|
19
|
+
"vendored",
|
|
20
|
+
"test",
|
|
21
|
+
"docs",
|
|
22
|
+
"changelog",
|
|
23
|
+
"lockfile",
|
|
24
|
+
"ci",
|
|
25
|
+
"packaging",
|
|
26
|
+
"other",
|
|
27
|
+
]
|
|
28
|
+
SOURCE_KINDS: frozenset[PathKind] = frozenset({"python", "native", "other_source"})
|
|
29
|
+
|
|
30
|
+
_VENDORED_DIRS = frozenset(
|
|
31
|
+
{"vendor", "vendored", "_vendor", "third_party", "thirdparty", "externals", "node_modules"}
|
|
32
|
+
)
|
|
33
|
+
_TEST_DIRS = frozenset({"test", "tests", "unittests", "unit_tests", "testdata"})
|
|
34
|
+
_DOC_DIRS = frozenset({"doc", "docs", "documentation", "example", "examples", "benchmarks"})
|
|
35
|
+
_CHANGELOG_DIRS = frozenset({"changelog.d", "newsfragments", "changes", "news"})
|
|
36
|
+
_CHANGELOG_STEMS = frozenset(
|
|
37
|
+
{"CHANGES", "CHANGELOG", "HISTORY", "NEWS", "RELEASE_NOTES", "RELEASES", "WHATSNEW"}
|
|
38
|
+
)
|
|
39
|
+
_CI_DIRS = frozenset({".github", ".circleci", ".gitlab", ".azure-pipelines"})
|
|
40
|
+
_CI_FILES = frozenset(
|
|
41
|
+
{
|
|
42
|
+
".gitlab-ci.yml",
|
|
43
|
+
".travis.yml",
|
|
44
|
+
"azure-pipelines.yml",
|
|
45
|
+
"tox.ini",
|
|
46
|
+
"noxfile.py",
|
|
47
|
+
"appveyor.yml",
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
_PACKAGING_FILES = frozenset(
|
|
51
|
+
{
|
|
52
|
+
"setup.py",
|
|
53
|
+
"setup.cfg",
|
|
54
|
+
"pyproject.toml",
|
|
55
|
+
"manifest.in",
|
|
56
|
+
"_version.py",
|
|
57
|
+
"version.py",
|
|
58
|
+
"__about__.py",
|
|
59
|
+
"__version__.py",
|
|
60
|
+
}
|
|
61
|
+
)
|
|
62
|
+
_LOCKFILE_NAMES = frozenset(
|
|
63
|
+
{"pipfile", "pipfile.lock", "poetry.lock", "uv.lock", "package-lock.json", "yarn.lock"}
|
|
64
|
+
)
|
|
65
|
+
_DOC_SUFFIXES = frozenset({".md", ".rst", ".txt", ".adoc"})
|
|
66
|
+
_PYTHON_SUFFIXES = frozenset({".py", ".pyi", ".pyx", ".pxd", ".pyw"})
|
|
67
|
+
_NATIVE_SUFFIXES = frozenset(
|
|
68
|
+
{".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".rs", ".go", ".m", ".f90"}
|
|
69
|
+
)
|
|
70
|
+
_OTHER_SOURCE_SUFFIXES = frozenset(
|
|
71
|
+
{".js", ".mjs", ".ts", ".tsx", ".jsx", ".html", ".htm", ".jinja", ".jinja2", ".j2",
|
|
72
|
+
".mako", ".xml", ".xsl", ".sql", ".sh", ".vue", ".css"}
|
|
73
|
+
) # fmt: skip
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _is_test_file(name: str) -> bool:
|
|
77
|
+
stem = name.rsplit(".", 1)[0]
|
|
78
|
+
return (
|
|
79
|
+
name == "conftest.py"
|
|
80
|
+
or name in ("tests.py", "test.py")
|
|
81
|
+
or (
|
|
82
|
+
name.endswith(".py")
|
|
83
|
+
and (stem.startswith("test_") or stem.endswith(("_test", "_tests")))
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def classify_path(path: str) -> PathKind:
|
|
89
|
+
"""Classify one repository-relative path from a diff."""
|
|
90
|
+
pure = PurePosixPath(path.strip("/"))
|
|
91
|
+
parts = [p.lower() for p in pure.parts]
|
|
92
|
+
directories = set(parts[:-1])
|
|
93
|
+
name = pure.name
|
|
94
|
+
lower = name.lower()
|
|
95
|
+
suffix = pure.suffix.lower()
|
|
96
|
+
stem_upper = name.split(".", 1)[0].upper()
|
|
97
|
+
|
|
98
|
+
if directories & _VENDORED_DIRS:
|
|
99
|
+
return "vendored"
|
|
100
|
+
if directories & _TEST_DIRS or _is_test_file(lower):
|
|
101
|
+
return "test"
|
|
102
|
+
if directories & _CI_DIRS or lower in _CI_FILES:
|
|
103
|
+
return "ci"
|
|
104
|
+
code_suffix = suffix in _PYTHON_SUFFIXES or suffix in _NATIVE_SUFFIXES
|
|
105
|
+
if (stem_upper in _CHANGELOG_STEMS or directories & _CHANGELOG_DIRS) and not code_suffix:
|
|
106
|
+
return "changelog"
|
|
107
|
+
if (
|
|
108
|
+
lower in _LOCKFILE_NAMES
|
|
109
|
+
or suffix == ".lock"
|
|
110
|
+
or (suffix == ".txt" and lower.startswith(("requirements", "constraints")))
|
|
111
|
+
):
|
|
112
|
+
return "lockfile"
|
|
113
|
+
if directories & _DOC_DIRS or suffix in _DOC_SUFFIXES:
|
|
114
|
+
return "docs"
|
|
115
|
+
if lower in _PACKAGING_FILES:
|
|
116
|
+
return "packaging"
|
|
117
|
+
if suffix in _PYTHON_SUFFIXES:
|
|
118
|
+
return "python"
|
|
119
|
+
if suffix in _NATIVE_SUFFIXES:
|
|
120
|
+
return "native"
|
|
121
|
+
if suffix in _OTHER_SOURCE_SUFFIXES:
|
|
122
|
+
return "other_source"
|
|
123
|
+
return "other"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def is_source(path: str) -> bool:
|
|
127
|
+
return classify_path(path) in SOURCE_KINDS
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""LLM extraction: prompt rendering, structured output, validation and repair."""
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""One extraction: render, call the model, validate, repair once, or dead-letter."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence, Set
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict
|
|
7
|
+
|
|
8
|
+
from patchsight.pipeline.extract.prompt import (
|
|
9
|
+
AdvisoryText,
|
|
10
|
+
CandidateInput,
|
|
11
|
+
PromptTemplate,
|
|
12
|
+
RenderedPrompt,
|
|
13
|
+
render,
|
|
14
|
+
)
|
|
15
|
+
from patchsight.pipeline.extract.schema import SCHEMA_NAME, response_schema
|
|
16
|
+
from patchsight.pipeline.extract.validate import (
|
|
17
|
+
ValidatedExtraction,
|
|
18
|
+
ValidationFailedError,
|
|
19
|
+
validate,
|
|
20
|
+
)
|
|
21
|
+
from patchsight.pipeline.llm.client import (
|
|
22
|
+
LLMClient,
|
|
23
|
+
LLMRequest,
|
|
24
|
+
LLMResponse,
|
|
25
|
+
ProviderRejectedError,
|
|
26
|
+
)
|
|
27
|
+
from patchsight.pipeline.llm.config import ModelConfig
|
|
28
|
+
from patchsight.pipeline.llm.tokens import TokenCounter
|
|
29
|
+
|
|
30
|
+
Status = Literal["ok", "repaired", "dead_letter", "no_candidates"]
|
|
31
|
+
|
|
32
|
+
REPAIR_MESSAGE = (
|
|
33
|
+
"Your answer failed validation:\n{errors}\n\n"
|
|
34
|
+
"Return corrected JSON for the same candidates. Use only the candidate and hunk IDs "
|
|
35
|
+
"shown above, and give exactly one item per candidate."
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ExtractionOutcome(BaseModel):
|
|
40
|
+
model_config = ConfigDict(frozen=True)
|
|
41
|
+
|
|
42
|
+
advisory_id: str
|
|
43
|
+
status: Status
|
|
44
|
+
model: str
|
|
45
|
+
prompt_version: str
|
|
46
|
+
candidates: int
|
|
47
|
+
vulnerable: tuple[str, ...]
|
|
48
|
+
entry_points: tuple[str, ...]
|
|
49
|
+
entry_points_dropped: int
|
|
50
|
+
invented_ids_first_attempt: int
|
|
51
|
+
errors: tuple[str, ...]
|
|
52
|
+
truncated: bool
|
|
53
|
+
dropped_candidates: tuple[str, ...]
|
|
54
|
+
estimated_prompt_tokens: int
|
|
55
|
+
prompt_tokens: int
|
|
56
|
+
completion_tokens: int
|
|
57
|
+
calls: int
|
|
58
|
+
cached_calls: int
|
|
59
|
+
provider_rejections: int
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _request(
|
|
63
|
+
model: ModelConfig, prompt: RenderedPrompt, messages: Sequence[dict[str, str]]
|
|
64
|
+
) -> LLMRequest:
|
|
65
|
+
return LLMRequest(
|
|
66
|
+
model=model.model,
|
|
67
|
+
prompt_version=prompt.version,
|
|
68
|
+
messages=tuple(messages),
|
|
69
|
+
schema_name=SCHEMA_NAME,
|
|
70
|
+
response_schema=response_schema(),
|
|
71
|
+
max_tokens=model.max_output_tokens,
|
|
72
|
+
seed=model.seed,
|
|
73
|
+
extra=model.extra,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def extract(
|
|
78
|
+
llm: LLMClient,
|
|
79
|
+
model: ModelConfig,
|
|
80
|
+
template: PromptTemplate,
|
|
81
|
+
advisory: AdvisoryText,
|
|
82
|
+
inputs: Sequence[CandidateInput],
|
|
83
|
+
known_names: Set[str],
|
|
84
|
+
budget_tokens: int,
|
|
85
|
+
counter: TokenCounter,
|
|
86
|
+
max_hunk_lines: int,
|
|
87
|
+
) -> ExtractionOutcome:
|
|
88
|
+
prompt = render(template, advisory, inputs, budget_tokens, counter, max_hunk_lines)
|
|
89
|
+
base = {
|
|
90
|
+
"advisory_id": advisory.id,
|
|
91
|
+
"model": model.model,
|
|
92
|
+
"prompt_version": prompt.version,
|
|
93
|
+
"candidates": len(prompt.candidates),
|
|
94
|
+
"truncated": prompt.truncated,
|
|
95
|
+
"dropped_candidates": prompt.dropped,
|
|
96
|
+
"estimated_prompt_tokens": prompt.estimated_tokens,
|
|
97
|
+
}
|
|
98
|
+
if not prompt.candidates:
|
|
99
|
+
return ExtractionOutcome(
|
|
100
|
+
**base, status="no_candidates", vulnerable=(), entry_points=(), entry_points_dropped=0,
|
|
101
|
+
invented_ids_first_attempt=0, errors=(), prompt_tokens=0, completion_tokens=0,
|
|
102
|
+
calls=0, cached_calls=0, provider_rejections=0,
|
|
103
|
+
) # fmt: skip
|
|
104
|
+
|
|
105
|
+
responses: list[LLMResponse] = []
|
|
106
|
+
messages = list(prompt.messages)
|
|
107
|
+
first_invented = 0
|
|
108
|
+
errors: list[str] = []
|
|
109
|
+
result: ValidatedExtraction | None = None
|
|
110
|
+
rejections = 0
|
|
111
|
+
for attempt in range(2):
|
|
112
|
+
try:
|
|
113
|
+
response = llm.complete(_request(model, prompt, messages))
|
|
114
|
+
except ProviderRejectedError as rejected:
|
|
115
|
+
rejections += 1
|
|
116
|
+
errors = [f"the provider rejected the generation: {rejected}"]
|
|
117
|
+
messages.append(
|
|
118
|
+
{"role": "user", "content": REPAIR_MESSAGE.format(errors="\n".join(errors))}
|
|
119
|
+
)
|
|
120
|
+
continue
|
|
121
|
+
responses.append(response)
|
|
122
|
+
try:
|
|
123
|
+
result = validate(response.content, prompt, known_names)
|
|
124
|
+
break
|
|
125
|
+
except ValidationFailedError as failure:
|
|
126
|
+
if attempt == 0:
|
|
127
|
+
first_invented = failure.invented_ids
|
|
128
|
+
errors = failure.errors
|
|
129
|
+
messages += [
|
|
130
|
+
{"role": "assistant", "content": response.content},
|
|
131
|
+
{"role": "user", "content": REPAIR_MESSAGE.format(errors="\n".join(errors))},
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
names = {c.id: c.name for c in prompt.candidates}
|
|
135
|
+
usage = {
|
|
136
|
+
"prompt_tokens": sum(r.prompt_tokens for r in responses),
|
|
137
|
+
"completion_tokens": sum(r.completion_tokens for r in responses),
|
|
138
|
+
"calls": len(responses) + rejections,
|
|
139
|
+
"cached_calls": sum(r.cached for r in responses),
|
|
140
|
+
"provider_rejections": rejections,
|
|
141
|
+
"invented_ids_first_attempt": first_invented,
|
|
142
|
+
}
|
|
143
|
+
if result is None:
|
|
144
|
+
return ExtractionOutcome(
|
|
145
|
+
**base, **usage, status="dead_letter", vulnerable=(), entry_points=(),
|
|
146
|
+
entry_points_dropped=0, errors=tuple(errors),
|
|
147
|
+
) # fmt: skip
|
|
148
|
+
return ExtractionOutcome(
|
|
149
|
+
**base,
|
|
150
|
+
**usage,
|
|
151
|
+
status="ok" if len(responses) == 1 and not rejections else "repaired",
|
|
152
|
+
vulnerable=tuple(sorted({names[cid] for cid in result.vulnerable_ids})),
|
|
153
|
+
entry_points=result.entry_points,
|
|
154
|
+
entry_points_dropped=len(result.entry_points_dropped),
|
|
155
|
+
errors=tuple(errors) if len(responses) > 1 or rejections else (),
|
|
156
|
+
)
|