patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
patchsight/__init__.py
ADDED
patchsight/cli.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""User-facing command-line interface.
|
|
2
|
+
|
|
3
|
+
This module is on the hot path: it must never import ``patchsight.pipeline``,
|
|
4
|
+
``patchsight.eval`` or any LLM client. ``tests/test_import_boundary.py`` enforces this.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from patchsight import __version__
|
|
13
|
+
|
|
14
|
+
app = typer.Typer(
|
|
15
|
+
name="patchsight",
|
|
16
|
+
help="Symbol-level vulnerability triage for Python dependencies.",
|
|
17
|
+
no_args_is_help=True,
|
|
18
|
+
add_completion=False,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _print_version(value: bool) -> None:
|
|
23
|
+
if value:
|
|
24
|
+
typer.echo(f"patchsight {__version__}")
|
|
25
|
+
raise typer.Exit
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@app.callback()
|
|
29
|
+
def main(
|
|
30
|
+
version: Annotated[
|
|
31
|
+
bool,
|
|
32
|
+
typer.Option(
|
|
33
|
+
"--version",
|
|
34
|
+
callback=_print_version,
|
|
35
|
+
is_eager=True,
|
|
36
|
+
help="Show the version and exit.",
|
|
37
|
+
),
|
|
38
|
+
] = False,
|
|
39
|
+
) -> None:
|
|
40
|
+
"""Symbol-level vulnerability triage for Python dependencies."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
db_app = typer.Typer(help="Download and inspect the symbol database.", no_args_is_help=True)
|
|
44
|
+
app.add_typer(db_app, name="db")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@db_app.command("update")
|
|
48
|
+
def db_update() -> None:
|
|
49
|
+
"""Download the latest published database if it is newer, verifying its SHA-256."""
|
|
50
|
+
from patchsight.db import DatabaseError
|
|
51
|
+
from patchsight.scan import dbcache
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
status, pointer = dbcache.update()
|
|
55
|
+
except (DatabaseError, OSError) as error:
|
|
56
|
+
typer.echo(f"error: {error}", err=True)
|
|
57
|
+
raise typer.Exit(code=2) from None
|
|
58
|
+
verb = "Downloaded" if status == "updated" else "Already have"
|
|
59
|
+
typer.echo(
|
|
60
|
+
f"{verb} database {pointer.db_version} ({pointer.records:,} records) "
|
|
61
|
+
f"at {dbcache.default_db_path()}"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@db_app.command("info")
|
|
66
|
+
def db_info(
|
|
67
|
+
database: Annotated[Path | None, typer.Option("--db", help="Database to describe.")] = None,
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Show which database a scan would use."""
|
|
70
|
+
from patchsight.db import DatabaseError, SymbolDatabase
|
|
71
|
+
from patchsight.scan import dbcache
|
|
72
|
+
from patchsight.scan.report import stale_warning
|
|
73
|
+
|
|
74
|
+
path = database or dbcache.default_db_path()
|
|
75
|
+
try:
|
|
76
|
+
opened = SymbolDatabase(path)
|
|
77
|
+
except DatabaseError as error:
|
|
78
|
+
typer.echo(f"error: {error}; run 'patchsight db update'", err=True)
|
|
79
|
+
raise typer.Exit(code=2) from None
|
|
80
|
+
meta = opened.meta
|
|
81
|
+
opened.close()
|
|
82
|
+
typer.echo(f"{path}")
|
|
83
|
+
for key in ("db_version", "schema_version", "builder", "built_at", "snapshot_sha256"):
|
|
84
|
+
typer.echo(f" {key}: {meta.get(key, '?')}")
|
|
85
|
+
if warning := stale_warning(meta):
|
|
86
|
+
typer.echo(f" warning: {warning}")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@app.command()
|
|
90
|
+
def scan(
|
|
91
|
+
path: Annotated[Path, typer.Argument(help="Project directory to scan.")] = Path(),
|
|
92
|
+
requirements_file: Annotated[
|
|
93
|
+
Path | None, typer.Option("--requirements", help="Lockfile (default: requirements.txt).")
|
|
94
|
+
] = None,
|
|
95
|
+
database: Annotated[
|
|
96
|
+
Path | None,
|
|
97
|
+
typer.Option("--db", help="Symbol database (default: the one 'db update' downloaded)."),
|
|
98
|
+
] = None,
|
|
99
|
+
output: Annotated[str, typer.Option("--format", help="text or json.")] = "text",
|
|
100
|
+
fail_on: Annotated[
|
|
101
|
+
str, typer.Option(help="Exit 1 when an alert is: called, unknown (or called), or never.")
|
|
102
|
+
] = "never",
|
|
103
|
+
) -> None:
|
|
104
|
+
"""Label every dependency alert as called, imported but not called, not imported, or unknown."""
|
|
105
|
+
from patchsight.db import DatabaseError
|
|
106
|
+
from patchsight.scan.dbcache import default_db_path
|
|
107
|
+
from patchsight.scan.osv_client import OsvError
|
|
108
|
+
from patchsight.scan.report import render_json, render_text
|
|
109
|
+
from patchsight.scan.run import run_scan
|
|
110
|
+
|
|
111
|
+
lockfile = requirements_file or path / "requirements.txt"
|
|
112
|
+
if not lockfile.exists():
|
|
113
|
+
typer.echo(f"error: no lockfile at {lockfile}; pass --requirements", err=True)
|
|
114
|
+
raise typer.Exit(code=2)
|
|
115
|
+
try:
|
|
116
|
+
alerts, dependencies, meta = run_scan(path, lockfile, database or default_db_path())
|
|
117
|
+
except DatabaseError as error:
|
|
118
|
+
typer.echo(f"error: {error}; run 'patchsight db update' or pass --db", err=True)
|
|
119
|
+
raise typer.Exit(code=2) from None
|
|
120
|
+
except OsvError as error:
|
|
121
|
+
typer.echo(f"error: {error}", err=True)
|
|
122
|
+
raise typer.Exit(code=2) from None
|
|
123
|
+
shown = lockfile.as_posix()
|
|
124
|
+
if output == "json":
|
|
125
|
+
typer.echo(render_json(alerts, dependencies, shown, meta))
|
|
126
|
+
else:
|
|
127
|
+
typer.echo(render_text(alerts, dependencies, shown, meta))
|
|
128
|
+
failing = {"called": {"CALLED"}, "unknown": {"CALLED", "UNKNOWN"}}.get(fail_on, set())
|
|
129
|
+
if any(a.verdict in failing for a in alerts):
|
|
130
|
+
raise typer.Exit(code=1)
|
patchsight/db.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""The symbol database: the one artefact that crosses from the offline pipeline to the scan.
|
|
2
|
+
|
|
3
|
+
A record maps one vulnerability (an alias group) and one package to the symbols that carry
|
|
4
|
+
the bug, with confidence, evidence and provenance. Every advisory ID and alias points at
|
|
5
|
+
its vulnerability, so the scanner can look up whatever ID OSV returns.
|
|
6
|
+
|
|
7
|
+
* ``high`` confidence: LLM extraction that passed validation and agreement checks.
|
|
8
|
+
* ``low`` confidence: the fallback (every changed function) or an unconfirmed extraction.
|
|
9
|
+
Low-confidence symbols can make an alert ``CALLED`` but never ``IMPORTED_NOT_CALLED``.
|
|
10
|
+
|
|
11
|
+
Stdlib ``sqlite3`` only, so the scan path can read it without any pipeline dependency.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import sqlite3
|
|
16
|
+
from collections.abc import Iterable
|
|
17
|
+
from datetime import datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Literal
|
|
20
|
+
|
|
21
|
+
from pydantic import BaseModel, ConfigDict
|
|
22
|
+
|
|
23
|
+
SCHEMA_VERSION = 1
|
|
24
|
+
|
|
25
|
+
Confidence = Literal["high", "low"]
|
|
26
|
+
Source = Literal["llm", "fallback", "override"]
|
|
27
|
+
|
|
28
|
+
_SCHEMA = """
|
|
29
|
+
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
30
|
+
CREATE TABLE aliases (identifier TEXT PRIMARY KEY, vulnerability TEXT NOT NULL);
|
|
31
|
+
CREATE TABLE records (
|
|
32
|
+
vulnerability TEXT NOT NULL,
|
|
33
|
+
package TEXT NOT NULL,
|
|
34
|
+
confidence TEXT NOT NULL CHECK (confidence IN ('high', 'low')),
|
|
35
|
+
source TEXT NOT NULL CHECK (source IN ('llm', 'fallback', 'override')),
|
|
36
|
+
import_roots TEXT NOT NULL,
|
|
37
|
+
entry_points TEXT NOT NULL,
|
|
38
|
+
fix_commits TEXT NOT NULL,
|
|
39
|
+
model TEXT,
|
|
40
|
+
prompt_version TEXT,
|
|
41
|
+
input_hash TEXT NOT NULL,
|
|
42
|
+
advisory_modified TEXT NOT NULL,
|
|
43
|
+
truncated INTEGER NOT NULL,
|
|
44
|
+
extracted_at TEXT NOT NULL,
|
|
45
|
+
PRIMARY KEY (vulnerability, package)
|
|
46
|
+
);
|
|
47
|
+
CREATE TABLE symbols (
|
|
48
|
+
vulnerability TEXT NOT NULL,
|
|
49
|
+
package TEXT NOT NULL,
|
|
50
|
+
qualified_name TEXT NOT NULL,
|
|
51
|
+
module TEXT,
|
|
52
|
+
name TEXT NOT NULL,
|
|
53
|
+
kind TEXT NOT NULL,
|
|
54
|
+
path TEXT NOT NULL,
|
|
55
|
+
FOREIGN KEY (vulnerability, package) REFERENCES records (vulnerability, package)
|
|
56
|
+
);
|
|
57
|
+
CREATE INDEX symbols_by_record ON symbols (vulnerability, package);
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class DatabaseError(RuntimeError):
|
|
62
|
+
"""The database is missing, of an unsupported schema version, or malformed."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class FixCommitRef(BaseModel):
|
|
66
|
+
model_config = ConfigDict(frozen=True)
|
|
67
|
+
|
|
68
|
+
repo: str
|
|
69
|
+
sha: str
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class Symbol(BaseModel):
|
|
73
|
+
"""A vulnerable definition. ``qualified_name`` is ``module.qualname`` when the module
|
|
74
|
+
is known, otherwise just the qualname."""
|
|
75
|
+
|
|
76
|
+
model_config = ConfigDict(frozen=True)
|
|
77
|
+
|
|
78
|
+
qualified_name: str
|
|
79
|
+
module: str | None
|
|
80
|
+
name: str
|
|
81
|
+
kind: str
|
|
82
|
+
path: str
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Record(BaseModel):
|
|
86
|
+
model_config = ConfigDict(frozen=True)
|
|
87
|
+
|
|
88
|
+
vulnerability: str
|
|
89
|
+
package: str
|
|
90
|
+
confidence: Confidence
|
|
91
|
+
source: Source
|
|
92
|
+
import_roots: tuple[str, ...]
|
|
93
|
+
entry_points: tuple[str, ...]
|
|
94
|
+
symbols: tuple[Symbol, ...]
|
|
95
|
+
fix_commits: tuple[FixCommitRef, ...]
|
|
96
|
+
model: str | None
|
|
97
|
+
prompt_version: str | None
|
|
98
|
+
input_hash: str
|
|
99
|
+
advisory_modified: datetime
|
|
100
|
+
truncated: bool
|
|
101
|
+
extracted_at: datetime
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def write(
|
|
105
|
+
path: Path, records: Iterable[Record], aliases: dict[str, str], meta: dict[str, str]
|
|
106
|
+
) -> int:
|
|
107
|
+
"""Write a fresh database file. Returns the number of records written."""
|
|
108
|
+
if path.exists():
|
|
109
|
+
path.unlink()
|
|
110
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
connection = sqlite3.connect(path)
|
|
112
|
+
count = 0
|
|
113
|
+
try:
|
|
114
|
+
connection.executescript(_SCHEMA)
|
|
115
|
+
connection.executemany(
|
|
116
|
+
"INSERT INTO meta VALUES (?, ?)",
|
|
117
|
+
sorted({**meta, "schema_version": str(SCHEMA_VERSION)}.items()),
|
|
118
|
+
)
|
|
119
|
+
connection.executemany("INSERT INTO aliases VALUES (?, ?)", sorted(aliases.items()))
|
|
120
|
+
for record in records:
|
|
121
|
+
count += 1
|
|
122
|
+
connection.execute(
|
|
123
|
+
"INSERT INTO records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
124
|
+
(
|
|
125
|
+
record.vulnerability,
|
|
126
|
+
record.package,
|
|
127
|
+
record.confidence,
|
|
128
|
+
record.source,
|
|
129
|
+
json.dumps(record.import_roots),
|
|
130
|
+
json.dumps(record.entry_points),
|
|
131
|
+
json.dumps([c.model_dump() for c in record.fix_commits]),
|
|
132
|
+
record.model,
|
|
133
|
+
record.prompt_version,
|
|
134
|
+
record.input_hash,
|
|
135
|
+
record.advisory_modified.isoformat(),
|
|
136
|
+
int(record.truncated),
|
|
137
|
+
record.extracted_at.isoformat(),
|
|
138
|
+
),
|
|
139
|
+
)
|
|
140
|
+
connection.executemany(
|
|
141
|
+
"INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
142
|
+
[
|
|
143
|
+
(
|
|
144
|
+
record.vulnerability,
|
|
145
|
+
record.package,
|
|
146
|
+
s.qualified_name,
|
|
147
|
+
s.module,
|
|
148
|
+
s.name,
|
|
149
|
+
s.kind,
|
|
150
|
+
s.path,
|
|
151
|
+
)
|
|
152
|
+
for s in record.symbols
|
|
153
|
+
],
|
|
154
|
+
)
|
|
155
|
+
connection.commit()
|
|
156
|
+
finally:
|
|
157
|
+
connection.close()
|
|
158
|
+
return count
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class SymbolDatabase:
|
|
162
|
+
"""Read-only access for the scanner."""
|
|
163
|
+
|
|
164
|
+
def __init__(self, path: Path) -> None:
|
|
165
|
+
if not path.exists():
|
|
166
|
+
raise DatabaseError(f"no symbol database at {path}")
|
|
167
|
+
self._connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
168
|
+
try:
|
|
169
|
+
self.meta = dict(self._connection.execute("SELECT key, value FROM meta").fetchall())
|
|
170
|
+
except sqlite3.DatabaseError as error:
|
|
171
|
+
raise DatabaseError(f"{path} is not a PatchSight database: {error}") from None
|
|
172
|
+
version = self.meta.get("schema_version")
|
|
173
|
+
if version != str(SCHEMA_VERSION):
|
|
174
|
+
raise DatabaseError(
|
|
175
|
+
f"{path} has schema version {version}; this version of PatchSight reads "
|
|
176
|
+
f"version {SCHEMA_VERSION}. Update PatchSight or download a matching database."
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
def close(self) -> None:
|
|
180
|
+
self._connection.close()
|
|
181
|
+
|
|
182
|
+
def record_count(self) -> int:
|
|
183
|
+
return int(self._connection.execute("SELECT COUNT(*) FROM records").fetchone()[0])
|
|
184
|
+
|
|
185
|
+
def vulnerability_for(self, identifier: str) -> str | None:
|
|
186
|
+
row = self._connection.execute(
|
|
187
|
+
"SELECT vulnerability FROM aliases WHERE identifier = ?", (identifier,)
|
|
188
|
+
).fetchone()
|
|
189
|
+
return str(row[0]) if row else None
|
|
190
|
+
|
|
191
|
+
def record(self, vulnerability: str, package: str) -> Record | None:
|
|
192
|
+
row = self._connection.execute(
|
|
193
|
+
"SELECT * FROM records WHERE vulnerability = ? AND package = ?",
|
|
194
|
+
(vulnerability, package),
|
|
195
|
+
).fetchone()
|
|
196
|
+
if row is None:
|
|
197
|
+
return None
|
|
198
|
+
symbols = self._connection.execute(
|
|
199
|
+
"SELECT qualified_name, module, name, kind, path FROM symbols "
|
|
200
|
+
"WHERE vulnerability = ? AND package = ? ORDER BY qualified_name",
|
|
201
|
+
(vulnerability, package),
|
|
202
|
+
).fetchall()
|
|
203
|
+
return Record(
|
|
204
|
+
vulnerability=row[0],
|
|
205
|
+
package=row[1],
|
|
206
|
+
confidence=row[2],
|
|
207
|
+
source=row[3],
|
|
208
|
+
import_roots=tuple(json.loads(row[4])),
|
|
209
|
+
entry_points=tuple(json.loads(row[5])),
|
|
210
|
+
fix_commits=tuple(FixCommitRef(**c) for c in json.loads(row[6])),
|
|
211
|
+
model=row[7],
|
|
212
|
+
prompt_version=row[8],
|
|
213
|
+
input_hash=row[9],
|
|
214
|
+
advisory_modified=datetime.fromisoformat(row[10]),
|
|
215
|
+
truncated=bool(row[11]),
|
|
216
|
+
extracted_at=datetime.fromisoformat(row[12]),
|
|
217
|
+
symbols=tuple(
|
|
218
|
+
Symbol(qualified_name=s[0], module=s[1], name=s[2], kind=s[3], path=s[4])
|
|
219
|
+
for s in symbols
|
|
220
|
+
),
|
|
221
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Evaluation harness: metrics, benchmark scoring and the release gate. Offline only."""
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Confidence tiers from two-model agreement, and the fallback union (proposal §6, step 5).
|
|
2
|
+
|
|
3
|
+
For each report, the extractor (primary) and a model of a different family (secondary)
|
|
4
|
+
answer independently. A report is **high** confidence only when all of these hold:
|
|
5
|
+
|
|
6
|
+
* both answers passed validation;
|
|
7
|
+
* the prompt was not truncated to the token budget;
|
|
8
|
+
* the symbol set is not empty, because "nothing is vulnerable" must never turn into
|
|
9
|
+
"not called" for every user;
|
|
10
|
+
* the two answers agree under the chosen rule;
|
|
11
|
+
* at least one agreed function or entry point is **public** (an exported Go name, or a
|
|
12
|
+
Python name without a leading underscore). A record whose symbols are all private
|
|
13
|
+
helpers cannot support "not called": users reach a helper only through public APIs
|
|
14
|
+
the record does not name.
|
|
15
|
+
|
|
16
|
+
Everything else is **low** confidence and, with the fallback union, predicts every changed
|
|
17
|
+
function. Uncertainty can therefore only add symbols, never remove them.
|
|
18
|
+
|
|
19
|
+
Agreement rules (open question 1). ``subset`` with the public-symbol condition was chosen on
|
|
20
|
+
the dev split (ADR 0004) and is the default:
|
|
21
|
+
|
|
22
|
+
* ``identical``: the same set; predicts it.
|
|
23
|
+
* ``subset``: one set contains the other; predicts their union.
|
|
24
|
+
* ``overlap``: Jaccard similarity of at least 0.5; predicts their union.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from collections import Counter
|
|
28
|
+
from collections.abc import Mapping, Set
|
|
29
|
+
from typing import Literal
|
|
30
|
+
|
|
31
|
+
from pydantic import BaseModel, ConfigDict
|
|
32
|
+
|
|
33
|
+
from patchsight.eval.metrics import SetScores, jaccard, score
|
|
34
|
+
from patchsight.eval.safety import SafetyScores, safety
|
|
35
|
+
|
|
36
|
+
AgreementRule = Literal["identical", "subset", "overlap"]
|
|
37
|
+
RULES: tuple[AgreementRule, ...] = ("identical", "subset", "overlap")
|
|
38
|
+
Tier = Literal["high", "low"]
|
|
39
|
+
VALID = frozenset({"ok", "repaired"})
|
|
40
|
+
CHOSEN_RULE: AgreementRule = "subset"
|
|
41
|
+
Language = Literal["go", "python"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def is_public(name: str, language: Language) -> bool:
|
|
45
|
+
"""Whether users can call this name directly: an exported Go identifier, or a Python
|
|
46
|
+
name without a leading underscore (dunder methods such as ``__init__`` count)."""
|
|
47
|
+
leaf = name.split(".")[-1]
|
|
48
|
+
if language == "go":
|
|
49
|
+
return leaf[:1].isupper()
|
|
50
|
+
return not leaf.startswith("_") or (leaf.startswith("__") and leaf.endswith("__"))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Answer(BaseModel):
|
|
54
|
+
"""One model's validated outcome for one report."""
|
|
55
|
+
|
|
56
|
+
model_config = ConfigDict(frozen=True)
|
|
57
|
+
|
|
58
|
+
status: str
|
|
59
|
+
vulnerable: frozenset[str]
|
|
60
|
+
truncated: bool = False
|
|
61
|
+
entry_points: frozenset[str] = frozenset()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Decision(BaseModel):
|
|
65
|
+
model_config = ConfigDict(frozen=True)
|
|
66
|
+
|
|
67
|
+
tier: Tier
|
|
68
|
+
reason: str
|
|
69
|
+
agreed: frozenset[str]
|
|
70
|
+
# Agreed functions plus both models' entry points: what the scanner would match.
|
|
71
|
+
symbols: frozenset[str] = frozenset()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def decide(
|
|
75
|
+
primary: Answer,
|
|
76
|
+
secondary: Answer | None,
|
|
77
|
+
rule: AgreementRule = CHOSEN_RULE,
|
|
78
|
+
require_public: bool = True,
|
|
79
|
+
language: Language = "go",
|
|
80
|
+
) -> Decision:
|
|
81
|
+
empty: frozenset[str] = frozenset()
|
|
82
|
+
if primary.status not in VALID:
|
|
83
|
+
return Decision(tier="low", reason="extractor answer failed validation", agreed=empty)
|
|
84
|
+
if secondary is None or secondary.status not in VALID:
|
|
85
|
+
return Decision(tier="low", reason="no valid second opinion", agreed=empty)
|
|
86
|
+
if primary.truncated or secondary.truncated:
|
|
87
|
+
return Decision(tier="low", reason="prompt truncated to the token budget", agreed=empty)
|
|
88
|
+
a, b = primary.vulnerable, secondary.vulnerable
|
|
89
|
+
if not a or not b:
|
|
90
|
+
return Decision(tier="low", reason="an empty symbol set", agreed=empty)
|
|
91
|
+
if rule == "identical":
|
|
92
|
+
agrees, agreed = a == b, a
|
|
93
|
+
elif rule == "subset":
|
|
94
|
+
agrees, agreed = a <= b or b <= a, a | b
|
|
95
|
+
else:
|
|
96
|
+
agrees, agreed = jaccard(a, b) >= 0.5, a | b
|
|
97
|
+
if not agrees:
|
|
98
|
+
return Decision(tier="low", reason="the two models disagree", agreed=empty)
|
|
99
|
+
symbols = agreed | primary.entry_points | secondary.entry_points
|
|
100
|
+
if require_public and not any(is_public(name, language) for name in symbols):
|
|
101
|
+
return Decision(
|
|
102
|
+
tier="low", reason="no public symbol among the agreed answers", agreed=empty
|
|
103
|
+
)
|
|
104
|
+
return Decision(
|
|
105
|
+
tier="high", reason=f"both models agree ({rule})", agreed=agreed, symbols=symbols
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class TierScores(BaseModel):
|
|
110
|
+
rule: AgreementRule
|
|
111
|
+
require_public: bool
|
|
112
|
+
fallback: bool
|
|
113
|
+
overall: SetScores
|
|
114
|
+
high_reports: int
|
|
115
|
+
high_share: float
|
|
116
|
+
high: SetScores
|
|
117
|
+
safety_overall: SafetyScores
|
|
118
|
+
safety_high: SafetyScores
|
|
119
|
+
low_reasons: dict[str, int]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def evaluate(
|
|
123
|
+
primary: Mapping[str, Answer],
|
|
124
|
+
secondary: Mapping[str, Answer],
|
|
125
|
+
changed: Mapping[str, Set[str]],
|
|
126
|
+
truth: Mapping[str, Set[str]],
|
|
127
|
+
rule: AgreementRule,
|
|
128
|
+
fallback: bool,
|
|
129
|
+
require_public: bool = False,
|
|
130
|
+
language: Language = "go",
|
|
131
|
+
) -> TierScores:
|
|
132
|
+
"""Score one configuration in two views.
|
|
133
|
+
|
|
134
|
+
* **Benchmark view** (``overall``, ``high``): functions only, exact names. Without the
|
|
135
|
+
fallback, low-confidence reports keep the extractor's own set.
|
|
136
|
+
* **Safety view** (``safety_*``): functions plus entry points, lenient names. High
|
|
137
|
+
confidence keeps the agreed symbols; low confidence keeps every changed function plus
|
|
138
|
+
both models' entry points.
|
|
139
|
+
"""
|
|
140
|
+
predicted: dict[str, set[str]] = {}
|
|
141
|
+
high: dict[str, set[str]] = {}
|
|
142
|
+
product: dict[str, set[str]] = {}
|
|
143
|
+
product_high: dict[str, set[str]] = {}
|
|
144
|
+
reasons: Counter[str] = Counter()
|
|
145
|
+
for report, answer in primary.items():
|
|
146
|
+
other = secondary.get(report)
|
|
147
|
+
decision = decide(answer, other, rule, require_public, language)
|
|
148
|
+
entry = set(answer.entry_points) | (set(other.entry_points) if other else set())
|
|
149
|
+
if decision.tier == "high":
|
|
150
|
+
predicted[report] = high[report] = set(decision.agreed)
|
|
151
|
+
product[report] = product_high[report] = set(decision.symbols)
|
|
152
|
+
else:
|
|
153
|
+
reasons[decision.reason] += 1
|
|
154
|
+
predicted[report] = set(changed[report]) if fallback else set(answer.vulnerable)
|
|
155
|
+
product[report] = (set(changed[report]) if fallback else set(answer.vulnerable)) | entry
|
|
156
|
+
reports = list(primary)
|
|
157
|
+
truth_of = {r: set(truth[r]) for r in reports}
|
|
158
|
+
return TierScores(
|
|
159
|
+
rule=rule,
|
|
160
|
+
require_public=require_public,
|
|
161
|
+
fallback=fallback,
|
|
162
|
+
overall=score(predicted, truth_of),
|
|
163
|
+
high_reports=len(high),
|
|
164
|
+
high_share=len(high) / len(reports) if reports else 0.0,
|
|
165
|
+
high=score(high, {r: truth_of[r] for r in high}),
|
|
166
|
+
safety_overall=safety(product, truth_of),
|
|
167
|
+
safety_high=safety(product_high, {r: truth_of[r] for r in product_high}),
|
|
168
|
+
low_reasons=dict(reasons.most_common()),
|
|
169
|
+
)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Set-based extraction metrics over advisories (proposal §10).
|
|
2
|
+
|
|
3
|
+
The unit is a (report, symbol name) pair. For each report, predicted symbols are compared
|
|
4
|
+
with the true vulnerable symbols:
|
|
5
|
+
|
|
6
|
+
* true positive: predicted and true;
|
|
7
|
+
* false positive: predicted, not true;
|
|
8
|
+
* false negative: true, not predicted. This includes reports with no prediction at all.
|
|
9
|
+
|
|
10
|
+
Precision, recall and F1 are micro-averaged over all pairs. Confidence intervals come
|
|
11
|
+
from a bootstrap that resamples whole reports, not pairs, because symbols within one report
|
|
12
|
+
are correlated.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import random
|
|
16
|
+
from collections.abc import Mapping, Set
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
|
|
19
|
+
from pydantic import BaseModel, ConfigDict
|
|
20
|
+
|
|
21
|
+
Predictions = Mapping[str, Set[str]]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Interval(BaseModel):
|
|
25
|
+
model_config = ConfigDict(frozen=True)
|
|
26
|
+
|
|
27
|
+
low: float
|
|
28
|
+
high: float
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SetScores(BaseModel):
|
|
32
|
+
model_config = ConfigDict(frozen=True)
|
|
33
|
+
|
|
34
|
+
reports: int
|
|
35
|
+
true_positives: int
|
|
36
|
+
false_positives: int
|
|
37
|
+
false_negatives: int
|
|
38
|
+
precision: float
|
|
39
|
+
recall: float
|
|
40
|
+
f1: float
|
|
41
|
+
miss_rate: float
|
|
42
|
+
exact_match: float
|
|
43
|
+
precision_ci95: Interval | None = None
|
|
44
|
+
recall_ci95: Interval | None = None
|
|
45
|
+
f1_ci95: Interval | None = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class _Counts:
|
|
50
|
+
tp: int
|
|
51
|
+
fp: int
|
|
52
|
+
fn: int
|
|
53
|
+
missed: bool
|
|
54
|
+
exact: bool
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _counts(predicted: Set[str], truth: Set[str]) -> _Counts:
|
|
58
|
+
tp = len(predicted & truth)
|
|
59
|
+
return _Counts(
|
|
60
|
+
tp=tp,
|
|
61
|
+
fp=len(predicted - truth),
|
|
62
|
+
fn=len(truth - predicted),
|
|
63
|
+
missed=bool(truth) and tp == 0,
|
|
64
|
+
exact=predicted == truth,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _prf(tp: int, fp: int, fn: int) -> tuple[float, float, float]:
|
|
69
|
+
precision = tp / (tp + fp) if tp + fp else 0.0
|
|
70
|
+
recall = tp / (tp + fn) if tp + fn else 0.0
|
|
71
|
+
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
|
|
72
|
+
return precision, recall, f1
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _percentile(values: list[float], q: float) -> float:
|
|
76
|
+
ordered = sorted(values)
|
|
77
|
+
position = q * (len(ordered) - 1)
|
|
78
|
+
low = int(position)
|
|
79
|
+
high = min(low + 1, len(ordered) - 1)
|
|
80
|
+
return ordered[low] + (ordered[high] - ordered[low]) * (position - low)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def score(
|
|
84
|
+
predicted: Predictions,
|
|
85
|
+
truth: Predictions,
|
|
86
|
+
bootstrap: int = 0,
|
|
87
|
+
seed: int = 0,
|
|
88
|
+
) -> SetScores:
|
|
89
|
+
"""Score predictions for every report in ``truth``. Extra predicted reports are ignored."""
|
|
90
|
+
reports = sorted(truth)
|
|
91
|
+
per_report = [_counts(predicted.get(r, frozenset()), truth[r]) for r in reports]
|
|
92
|
+
tp = sum(c.tp for c in per_report)
|
|
93
|
+
fp = sum(c.fp for c in per_report)
|
|
94
|
+
fn = sum(c.fn for c in per_report)
|
|
95
|
+
precision, recall, f1 = _prf(tp, fp, fn)
|
|
96
|
+
with_truth = [c for c, r in zip(per_report, reports, strict=True) if truth[r]]
|
|
97
|
+
|
|
98
|
+
intervals: dict[str, Interval | None] = {
|
|
99
|
+
"precision_ci95": None,
|
|
100
|
+
"recall_ci95": None,
|
|
101
|
+
"f1_ci95": None,
|
|
102
|
+
}
|
|
103
|
+
if bootstrap and per_report:
|
|
104
|
+
rng = random.Random(seed) # noqa: S311 - statistical resampling, not security
|
|
105
|
+
samples: list[tuple[float, float, float]] = []
|
|
106
|
+
for _ in range(bootstrap):
|
|
107
|
+
draw = [per_report[rng.randrange(len(per_report))] for _ in per_report]
|
|
108
|
+
samples.append(
|
|
109
|
+
_prf(sum(c.tp for c in draw), sum(c.fp for c in draw), sum(c.fn for c in draw))
|
|
110
|
+
)
|
|
111
|
+
for index, key in enumerate(("precision_ci95", "recall_ci95", "f1_ci95")):
|
|
112
|
+
values = [s[index] for s in samples]
|
|
113
|
+
intervals[key] = Interval(
|
|
114
|
+
low=_percentile(values, 0.025), high=_percentile(values, 0.975)
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
return SetScores(
|
|
118
|
+
reports=len(reports),
|
|
119
|
+
true_positives=tp,
|
|
120
|
+
false_positives=fp,
|
|
121
|
+
false_negatives=fn,
|
|
122
|
+
precision=precision,
|
|
123
|
+
recall=recall,
|
|
124
|
+
f1=f1,
|
|
125
|
+
miss_rate=sum(c.missed for c in with_truth) / len(with_truth) if with_truth else 0.0,
|
|
126
|
+
exact_match=sum(c.exact for c in per_report) / len(per_report) if per_report else 0.0,
|
|
127
|
+
**intervals,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def jaccard(a: Set[str], b: Set[str]) -> float:
|
|
132
|
+
"""Overlap of two sets; 1.0 when both are empty."""
|
|
133
|
+
union = a | b
|
|
134
|
+
return len(a & b) / len(union) if union else 1.0
|