patchsight 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchsight-0.2.0 → patchsight-0.3.0}/PKG-INFO +2 -2
- {patchsight-0.2.0 → patchsight-0.3.0}/docs/pypi.md +1 -1
- {patchsight-0.2.0 → patchsight-0.3.0}/pyproject.toml +1 -1
- {patchsight-0.2.0 → patchsight-0.3.0}/pyproject.toml.orig +1 -1
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/db.py +58 -6
- patchsight-0.3.0/src/patchsight/eval/failures.py +118 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/__main__.py +297 -3
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/build_db.py +18 -5
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/prompt.py +16 -5
- patchsight-0.3.0/src/patchsight/pipeline/gate.py +153 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/client.py +18 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/pypi_extract.py +9 -6
- patchsight-0.3.0/src/patchsight/pipeline/update_db.py +124 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/dbcache.py +19 -9
- {patchsight-0.2.0 → patchsight-0.3.0}/LICENSE +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/cli.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/eval/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/eval/confidence.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/eval/metrics.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/eval/parity.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/eval/safety.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/names.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/bundle.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/changedfuncs.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/dunlap.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/extraction.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/replay.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/splits.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/commits.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/github.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/hunks.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/mapping.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/paths.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/extractor.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/schema.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/validate.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/fix_sample.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/http.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/aliases.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/ghsa_repo.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/osv.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/references.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/snapshot.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/kaggle_inputs.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/config.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/tokens.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm_records.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/pypi.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/pipeline/week1.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/py.typed +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/ast_scanner.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/explain.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/lockfiles/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/lockfiles/poetry.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/lockfiles/requirements.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/lockfiles/toml_lock.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/lockfiles/uv.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/osv_client.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/report.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/run.py +0 -0
- {patchsight-0.2.0 → patchsight-0.3.0}/src/patchsight/scan/verdicts.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchsight
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Know which vulnerable Python dependencies your code actually calls: symbol-level triage, no LLM at scan time.
|
|
5
5
|
Keywords: security,vulnerability,dependencies,supply-chain,sca,static-analysis,reachability,osv,pypi,cli
|
|
6
6
|
Author: Niroj33
|
|
@@ -93,7 +93,7 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
93
93
|
## Status: early release
|
|
94
94
|
|
|
95
95
|
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
96
|
-
- The
|
|
96
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. Expect many UNKNOWN verdicts until high-confidence records are published.
|
|
97
97
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
98
98
|
|
|
99
99
|
## Links
|
|
@@ -55,7 +55,7 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
55
55
|
## Status: early release
|
|
56
56
|
|
|
57
57
|
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
58
|
-
- The
|
|
58
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. Expect many UNKNOWN verdicts until high-confidence records are published.
|
|
59
59
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
60
60
|
|
|
61
61
|
## Links
|
|
@@ -10,11 +10,12 @@ its vulnerability, so the scanner can look up whatever ID OSV returns.
|
|
|
10
10
|
|
|
11
11
|
Stdlib ``sqlite3`` only, so the scan path can read it without any pipeline dependency.
|
|
12
12
|
|
|
13
|
-
Compatibility:
|
|
14
|
-
and the ``evidence``
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
Compatibility: three tables were added within schema version 1. ``extraction`` holds the
|
|
14
|
+
second-opinion model and the reason for the tier; ``evidence`` holds each flagged symbol's
|
|
15
|
+
rationale and cited hunk locations; ``builds`` is the ledger of every vulnerability
|
|
16
|
+
considered, with its input hash, for incremental updates. Readers that predate them never
|
|
17
|
+
query them, so they keep working; this reader treats a database without them as having no
|
|
18
|
+
such data. Evidence holds commit SHAs, paths and line ranges only, never code (rule 13).
|
|
18
19
|
"""
|
|
19
20
|
|
|
20
21
|
import json
|
|
@@ -68,6 +69,11 @@ CREATE TABLE extraction (
|
|
|
68
69
|
tier_reason TEXT,
|
|
69
70
|
PRIMARY KEY (vulnerability, package)
|
|
70
71
|
);
|
|
72
|
+
CREATE TABLE builds (
|
|
73
|
+
vulnerability TEXT PRIMARY KEY,
|
|
74
|
+
input_hash TEXT NOT NULL,
|
|
75
|
+
status TEXT NOT NULL
|
|
76
|
+
);
|
|
71
77
|
CREATE TABLE evidence (
|
|
72
78
|
vulnerability TEXT NOT NULL,
|
|
73
79
|
package TEXT NOT NULL,
|
|
@@ -90,6 +96,17 @@ class FixCommitRef(BaseModel):
|
|
|
90
96
|
sha: str
|
|
91
97
|
|
|
92
98
|
|
|
99
|
+
class BuildEntry(BaseModel):
|
|
100
|
+
"""One vulnerability the builder considered, with or without a record: the ledger an
|
|
101
|
+
incremental update compares against (ADR 0001: re-extract only when inputs change)."""
|
|
102
|
+
|
|
103
|
+
model_config = ConfigDict(frozen=True)
|
|
104
|
+
|
|
105
|
+
vulnerability: str
|
|
106
|
+
input_hash: str
|
|
107
|
+
status: str
|
|
108
|
+
|
|
109
|
+
|
|
93
110
|
class HunkLocation(BaseModel):
|
|
94
111
|
"""Where a cited diff hunk sits: the fix commit, the file and both line ranges."""
|
|
95
112
|
|
|
@@ -152,7 +169,11 @@ class Record(BaseModel):
|
|
|
152
169
|
|
|
153
170
|
|
|
154
171
|
def write(
|
|
155
|
-
path: Path,
|
|
172
|
+
path: Path,
|
|
173
|
+
records: Iterable[Record],
|
|
174
|
+
aliases: dict[str, str],
|
|
175
|
+
meta: dict[str, str],
|
|
176
|
+
builds: Iterable[BuildEntry] = (),
|
|
156
177
|
) -> int:
|
|
157
178
|
"""Write a fresh database file. Returns the number of records written."""
|
|
158
179
|
if path.exists():
|
|
@@ -167,6 +188,10 @@ def write(
|
|
|
167
188
|
sorted({**meta, "schema_version": str(SCHEMA_VERSION)}.items()),
|
|
168
189
|
)
|
|
169
190
|
connection.executemany("INSERT INTO aliases VALUES (?, ?)", sorted(aliases.items()))
|
|
191
|
+
connection.executemany(
|
|
192
|
+
"INSERT INTO builds VALUES (?, ?, ?)",
|
|
193
|
+
sorted((b.vulnerability, b.input_hash, b.status) for b in builds),
|
|
194
|
+
)
|
|
170
195
|
for record in records:
|
|
171
196
|
count += 1
|
|
172
197
|
connection.execute(
|
|
@@ -256,6 +281,7 @@ class SymbolDatabase:
|
|
|
256
281
|
}
|
|
257
282
|
self._has_extraction = "extraction" in tables
|
|
258
283
|
self._has_evidence = "evidence" in tables
|
|
284
|
+
self._has_builds = "builds" in tables
|
|
259
285
|
|
|
260
286
|
def close(self) -> None:
|
|
261
287
|
self._connection.close()
|
|
@@ -269,6 +295,32 @@ class SymbolDatabase:
|
|
|
269
295
|
).fetchone()
|
|
270
296
|
return str(row[0]) if row else None
|
|
271
297
|
|
|
298
|
+
def confidence_counts(self) -> dict[str, int]:
|
|
299
|
+
"""Records by confidence tier."""
|
|
300
|
+
rows = self._connection.execute(
|
|
301
|
+
"SELECT confidence, COUNT(*) FROM records GROUP BY confidence ORDER BY confidence"
|
|
302
|
+
)
|
|
303
|
+
return {str(r[0]): int(r[1]) for r in rows}
|
|
304
|
+
|
|
305
|
+
def vulnerabilities(self) -> list[str]:
|
|
306
|
+
"""Every vulnerability that has at least one record."""
|
|
307
|
+
rows = self._connection.execute(
|
|
308
|
+
"SELECT DISTINCT vulnerability FROM records ORDER BY vulnerability"
|
|
309
|
+
)
|
|
310
|
+
return [str(r[0]) for r in rows]
|
|
311
|
+
|
|
312
|
+
def alias_map(self) -> dict[str, str]:
|
|
313
|
+
"""Every advisory ID and alias, mapped to its vulnerability."""
|
|
314
|
+
rows = self._connection.execute("SELECT identifier, vulnerability FROM aliases")
|
|
315
|
+
return {str(r[0]): str(r[1]) for r in rows}
|
|
316
|
+
|
|
317
|
+
def build_ledger(self) -> dict[str, BuildEntry]:
|
|
318
|
+
"""Every vulnerability the builder considered; empty for an older database."""
|
|
319
|
+
if not self._has_builds:
|
|
320
|
+
return {}
|
|
321
|
+
rows = self._connection.execute("SELECT vulnerability, input_hash, status FROM builds")
|
|
322
|
+
return {r[0]: BuildEntry(vulnerability=r[0], input_hash=r[1], status=r[2]) for r in rows}
|
|
323
|
+
|
|
272
324
|
def identifiers_for(self, vulnerability: str) -> list[str]:
|
|
273
325
|
"""Every advisory ID and alias that points at this vulnerability."""
|
|
274
326
|
rows = self._connection.execute(
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Failure causes for extraction errors (proposal §10, "failure analysis").
|
|
2
|
+
|
|
3
|
+
Every missed true symbol (a false negative) gets exactly one cause, checked in this order:
|
|
4
|
+
|
|
5
|
+
* ``not_changed``: the fix did not change the symbol, so no patch-based method can find it
|
|
6
|
+
(the ceiling ChangedFuncs also hits: "vulnerable function unchanged, callee changed");
|
|
7
|
+
* ``name_format``: the fix changed a definition that matches the symbol only on a dotted
|
|
8
|
+
boundary (GoVulnDB's ``decryptTicket`` for the method ``Conn.decryptTicket``): a naming
|
|
9
|
+
difference in the ground truth, not a missed function;
|
|
10
|
+
* ``dead_letter``: the symbol was changed, but the extraction failed;
|
|
11
|
+
* ``truncated``: changed, but dropped from the prompt to fit the token budget;
|
|
12
|
+
* ``model_missed``: shown to the model and judged not vulnerable.
|
|
13
|
+
|
|
14
|
+
Every false positive is either ``name_format`` (it matches a true symbol on a dotted
|
|
15
|
+
boundary) or ``model_false_positive``.
|
|
16
|
+
|
|
17
|
+
A report-level miss (no true symbol found: the safety metric) is attributed to its most
|
|
18
|
+
actionable cause: a shown-but-missed symbol first, then truncation, then the rest.
|
|
19
|
+
|
|
20
|
+
Not separated: "the fix adds a new helper". The replay data does not record which side of
|
|
21
|
+
the diff a candidate comes from, so such cases fall under ``model_false_positive``.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from collections import Counter
|
|
25
|
+
from collections.abc import Mapping, Set
|
|
26
|
+
|
|
27
|
+
from pydantic import BaseModel, ConfigDict
|
|
28
|
+
|
|
29
|
+
from patchsight.eval.safety import lenient_match
|
|
30
|
+
|
|
31
|
+
MISS_CAUSES = ("not_changed", "name_format", "dead_letter", "truncated", "model_missed")
|
|
32
|
+
REPORT_MISS_ORDER = ("model_missed", "truncated", "dead_letter", "name_format", "not_changed")
|
|
33
|
+
VALID = frozenset({"ok", "repaired"})
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ReportInput(BaseModel):
|
|
37
|
+
model_config = ConfigDict(frozen=True)
|
|
38
|
+
|
|
39
|
+
truth: frozenset[str]
|
|
40
|
+
changed: frozenset[str] # every changed function (ChangedFuncs)
|
|
41
|
+
shown: frozenset[str] # the candidates the prompt kept
|
|
42
|
+
status: str
|
|
43
|
+
predicted: frozenset[str]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class FailureBreakdown(BaseModel):
|
|
47
|
+
model_config = ConfigDict(frozen=True)
|
|
48
|
+
|
|
49
|
+
reports: int
|
|
50
|
+
truth_symbols: int
|
|
51
|
+
false_negatives: dict[str, int]
|
|
52
|
+
false_positives: dict[str, int]
|
|
53
|
+
report_misses: dict[str, int]
|
|
54
|
+
examples: dict[str, tuple[str, ...]] # cause -> report IDs (sorted)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def miss_cause(symbol: str, report: ReportInput) -> str:
|
|
58
|
+
if symbol not in report.changed:
|
|
59
|
+
if any(lenient_match(c, symbol) for c in report.changed):
|
|
60
|
+
return "name_format"
|
|
61
|
+
return "not_changed"
|
|
62
|
+
if report.status not in VALID:
|
|
63
|
+
return "dead_letter"
|
|
64
|
+
if symbol not in report.shown:
|
|
65
|
+
return "truncated"
|
|
66
|
+
return "model_missed"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def breakdown(reports: Mapping[str, ReportInput]) -> FailureBreakdown:
|
|
70
|
+
negatives: Counter[str] = Counter()
|
|
71
|
+
positives: Counter[str] = Counter()
|
|
72
|
+
misses: Counter[str] = Counter()
|
|
73
|
+
examples: dict[str, set[str]] = {}
|
|
74
|
+
total = 0
|
|
75
|
+
for rid, report in sorted(reports.items()):
|
|
76
|
+
total += len(report.truth)
|
|
77
|
+
causes = {miss_cause(s, report) for s in report.truth - report.predicted}
|
|
78
|
+
for symbol in report.truth - report.predicted:
|
|
79
|
+
cause = miss_cause(symbol, report)
|
|
80
|
+
negatives[cause] += 1
|
|
81
|
+
examples.setdefault(cause, set()).add(rid)
|
|
82
|
+
for symbol in report.predicted - report.truth:
|
|
83
|
+
cause = (
|
|
84
|
+
"name_format"
|
|
85
|
+
if any(lenient_match(symbol, t) for t in report.truth)
|
|
86
|
+
else "model_false_positive"
|
|
87
|
+
)
|
|
88
|
+
positives[cause] += 1
|
|
89
|
+
examples.setdefault(f"fp:{cause}", set()).add(rid)
|
|
90
|
+
if report.truth and not report.truth & report.predicted:
|
|
91
|
+
misses[next(c for c in REPORT_MISS_ORDER if c in causes)] += 1
|
|
92
|
+
return FailureBreakdown(
|
|
93
|
+
reports=len(reports),
|
|
94
|
+
truth_symbols=total,
|
|
95
|
+
false_negatives={c: negatives[c] for c in MISS_CAUSES},
|
|
96
|
+
false_positives={c: positives[c] for c in ("name_format", "model_false_positive")},
|
|
97
|
+
report_misses={c: misses[c] for c in REPORT_MISS_ORDER},
|
|
98
|
+
examples={k: tuple(sorted(v)) for k, v in sorted(examples.items())},
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def as_inputs(
|
|
103
|
+
truth: Mapping[str, Set[str]],
|
|
104
|
+
changed: Mapping[str, Set[str]],
|
|
105
|
+
shown: Mapping[str, Set[str]],
|
|
106
|
+
status: Mapping[str, str],
|
|
107
|
+
predicted: Mapping[str, Set[str]],
|
|
108
|
+
) -> dict[str, ReportInput]:
|
|
109
|
+
return {
|
|
110
|
+
r: ReportInput(
|
|
111
|
+
truth=frozenset(truth[r]),
|
|
112
|
+
changed=frozenset(changed.get(r, set())),
|
|
113
|
+
shown=frozenset(shown.get(r, set())),
|
|
114
|
+
status=status.get(r, "dead_letter"),
|
|
115
|
+
predicted=frozenset(predicted.get(r, set())),
|
|
116
|
+
)
|
|
117
|
+
for r in truth
|
|
118
|
+
}
|
|
@@ -30,10 +30,16 @@ from patchsight.pipeline.extract.prompt import load_template
|
|
|
30
30
|
from patchsight.pipeline.ingest import ghsa_repo, snapshot
|
|
31
31
|
from patchsight.pipeline.ingest.aliases import AliasGroup, alias_groups
|
|
32
32
|
from patchsight.pipeline.ingest.osv import Advisory, Excluded, Quarantined, normalise_dump
|
|
33
|
-
from patchsight.pipeline.llm.client import
|
|
33
|
+
from patchsight.pipeline.llm.client import (
|
|
34
|
+
QuotaExhaustedError,
|
|
35
|
+
RateLimitedError,
|
|
36
|
+
cache_only_client,
|
|
37
|
+
client_for,
|
|
38
|
+
)
|
|
34
39
|
from patchsight.pipeline.llm.config import load_config
|
|
35
40
|
from patchsight.pipeline.llm.tokens import HfTokenizer
|
|
36
41
|
from patchsight.pipeline.pypi import PyPIClient
|
|
42
|
+
from patchsight.scan import dbcache
|
|
37
43
|
|
|
38
44
|
app = typer.Typer(name="patchsight-pipeline", no_args_is_help=True, add_completion=False)
|
|
39
45
|
|
|
@@ -535,7 +541,10 @@ def build_db_command(
|
|
|
535
541
|
if llm is not None:
|
|
536
542
|
meta["records_by_tier"] = json.dumps(dict(Counter(r.confidence for r in records)))
|
|
537
543
|
target = root / out
|
|
538
|
-
db.write(
|
|
544
|
+
db.write(
|
|
545
|
+
target.with_name(target.name + ".sqlite"), records, aliases, meta,
|
|
546
|
+
builds=build_db.ledger(report.outcomes),
|
|
547
|
+
) # fmt: skip
|
|
539
548
|
target.with_name(target.name + ".jsonl").write_text(build_db.jsonl(records), "utf-8")
|
|
540
549
|
target.with_name(target.name + "-build.json").write_text(
|
|
541
550
|
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
@@ -809,6 +818,95 @@ def parity_report_command(
|
|
|
809
818
|
typer.echo("\n".join(lines[9:]))
|
|
810
819
|
|
|
811
820
|
|
|
821
|
+
@app.command("failure-report")
|
|
822
|
+
def failure_report_command(
|
|
823
|
+
tag: Annotated[str, typer.Argument(help="Run tag of a committed replay bundle.")],
|
|
824
|
+
part: Annotated[str, typer.Option("--split", help="Split part: dev, test or all.")] = "all",
|
|
825
|
+
root: Root = Path(),
|
|
826
|
+
) -> None:
|
|
827
|
+
"""Tag every extraction error with a cause, from committed files (no network)."""
|
|
828
|
+
from patchsight.eval import failures
|
|
829
|
+
|
|
830
|
+
data = bundle.load(root / DUNLAP_BUNDLE)
|
|
831
|
+
frozen = splits.load_split(root / DUNLAP_SPLIT)
|
|
832
|
+
ids = frozen.split.ids(part)
|
|
833
|
+
run = replay.load(root / DUNLAP_BUNDLE.parent / f"extraction-{tag}.json")
|
|
834
|
+
answered = {r.id: r for r in run.reports if r.id in ids}
|
|
835
|
+
truth = {r.id: set(r.symbols) for r in data.reports if r.id in ids}
|
|
836
|
+
changed = {r.id: r.changed_functions() for r in data.reports if r.id in ids}
|
|
837
|
+
shown = {rid: {c.name for c in r.candidates} for rid, r in answered.items()}
|
|
838
|
+
status: dict[str, str] = {}
|
|
839
|
+
predicted: dict[str, set[str]] = {}
|
|
840
|
+
for rid, report in answered.items():
|
|
841
|
+
status[rid], predicted[rid] = replay.replay_report(report, run.prompt_version)
|
|
842
|
+
result = failures.breakdown(failures.as_inputs(truth, changed, shown, status, predicted))
|
|
843
|
+
fn_total = sum(result.false_negatives.values())
|
|
844
|
+
fp_total = sum(result.false_positives.values())
|
|
845
|
+
miss_total = sum(result.report_misses.values())
|
|
846
|
+
labels = {
|
|
847
|
+
"dead_letter": "Extraction failed (dead letter)",
|
|
848
|
+
"not_changed": "Not changed by the fix (patch ceiling)",
|
|
849
|
+
"name_format": "Name format only (dotted-boundary match)",
|
|
850
|
+
"truncated": "Dropped to fit the token budget",
|
|
851
|
+
"model_missed": "Shown to the model, judged not vulnerable",
|
|
852
|
+
"model_false_positive": "Judged vulnerable, not in the ground truth",
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
def share(n: int, total: int) -> str:
|
|
856
|
+
return f"{n} ({n / total:.0%})" if total else "0"
|
|
857
|
+
|
|
858
|
+
lines = [
|
|
859
|
+
f"# Failure causes on the Dunlap `{part}` split: `{tag}`",
|
|
860
|
+
"",
|
|
861
|
+
"Generated by `python -m patchsight.pipeline failure-report`; do not edit by hand. "
|
|
862
|
+
"Computed offline from the committed candidate and replay bundles.",
|
|
863
|
+
"",
|
|
864
|
+
f"- Model: `{run.model}`, prompt `{run.prompt_version}`",
|
|
865
|
+
f"- Split `{frozen.split.name}:{part}` (SHA-256 `{frozen.sha256}`): {result.reports} "
|
|
866
|
+
f"reports, {result.truth_symbols} true symbols",
|
|
867
|
+
"",
|
|
868
|
+
"Causes are assigned in a fixed order; see `src/patchsight/eval/failures.py`. "
|
|
869
|
+
'"The fix adds a new helper" cannot be separated from this data and falls under '
|
|
870
|
+
"false positives.",
|
|
871
|
+
"",
|
|
872
|
+
f"## Missed true symbols ({fn_total})",
|
|
873
|
+
"",
|
|
874
|
+
"| Cause | Symbols |",
|
|
875
|
+
"|---|---|",
|
|
876
|
+
*(f"| {labels[c]} | {share(n, fn_total)} |" for c, n in result.false_negatives.items()),
|
|
877
|
+
"",
|
|
878
|
+
f"## Reports with no true symbol found ({miss_total}): the safety metric",
|
|
879
|
+
"",
|
|
880
|
+
"Each is attributed to its most actionable cause.",
|
|
881
|
+
"",
|
|
882
|
+
"| Cause | Reports |",
|
|
883
|
+
"|---|---|",
|
|
884
|
+
*(f"| {labels[c]} | {share(n, miss_total)} |" for c, n in result.report_misses.items()),
|
|
885
|
+
"",
|
|
886
|
+
f"## False positives ({fp_total})",
|
|
887
|
+
"",
|
|
888
|
+
"| Cause | Symbols |",
|
|
889
|
+
"|---|---|",
|
|
890
|
+
*(f"| {labels[c]} | {share(n, fp_total)} |" for c, n in result.false_positives.items()),
|
|
891
|
+
"",
|
|
892
|
+
"## Reports by cause",
|
|
893
|
+
"",
|
|
894
|
+
*(
|
|
895
|
+
f"- {labels.get(k.removeprefix('fp:'), k)}"
|
|
896
|
+
f"{' (false positive)' if k.startswith('fp:') else ''}: "
|
|
897
|
+
+ ", ".join(f"`{r}`" for r in v)
|
|
898
|
+
for k, v in result.examples.items()
|
|
899
|
+
),
|
|
900
|
+
"",
|
|
901
|
+
]
|
|
902
|
+
target = root / "docs" / "reports" / f"failures-dunlap-{part}-{tag}"
|
|
903
|
+
target.with_name(target.name + ".md").write_text("\n".join(lines), "utf-8")
|
|
904
|
+
target.with_name(target.name + ".json").write_text(
|
|
905
|
+
result.model_dump_json(indent=2) + "\n", "utf-8"
|
|
906
|
+
)
|
|
907
|
+
typer.echo("\n".join(lines[10:36]))
|
|
908
|
+
|
|
909
|
+
|
|
812
910
|
@app.command("reproduce")
|
|
813
911
|
def reproduce_command(root: Root = Path()) -> None:
|
|
814
912
|
"""Recompute the benchmark tables offline from committed files. No network, no API key."""
|
|
@@ -866,6 +964,195 @@ def reproduce_command(root: Root = Path()) -> None:
|
|
|
866
964
|
typer.echo(f"\nMatches docs/reports/extraction-dunlap-*-{tag}.json exactly.")
|
|
867
965
|
|
|
868
966
|
|
|
967
|
+
@app.command("update-db")
|
|
968
|
+
def update_db_command(
|
|
969
|
+
previous: Annotated[Path, typer.Argument(help="The previous database (SQLite).")],
|
|
970
|
+
root: Root = Path(),
|
|
971
|
+
cache: Annotated[Path, typer.Option(help="HTTP response cache.")] = Path(".cache/http"),
|
|
972
|
+
out: Annotated[Path, typer.Option(help="Output path, without extension.")] = Path(
|
|
973
|
+
"data/build/patchsight-update"
|
|
974
|
+
),
|
|
975
|
+
llm: Annotated[
|
|
976
|
+
Path | None, typer.Option(help="Extractor outcomes for the rebuilt vulnerabilities.")
|
|
977
|
+
] = None,
|
|
978
|
+
second: Annotated[Path | None, typer.Option(help="Second-opinion outcomes for them.")] = None,
|
|
979
|
+
) -> None:
|
|
980
|
+
"""Update a database from the pinned snapshot, rebuilding only changed vulnerabilities
|
|
981
|
+
(needs GITHUB_TOKEN)."""
|
|
982
|
+
from patchsight.pipeline import llm_records, update_db
|
|
983
|
+
|
|
984
|
+
token = _github_token()
|
|
985
|
+
manifest, dump = _load_verified(root)
|
|
986
|
+
by_id, groups = _github_fix_groups(dump, None, None, 0)
|
|
987
|
+
dump_entry = manifest.file(DUMP)
|
|
988
|
+
version = dump_entry.retrieved_at.strftime("%Y.%m.%d")
|
|
989
|
+
prior = db.SymbolDatabase(root / previous)
|
|
990
|
+
prior_meta = dict(prior.meta)
|
|
991
|
+
github = GitHubClient(root / cache / "github", token=token, wait_for_rate_limit=3900)
|
|
992
|
+
pypi = PyPIClient(root / cache / "pypi")
|
|
993
|
+
try:
|
|
994
|
+
records, aliases, builds, report = update_db.update(
|
|
995
|
+
prior, groups, by_id, github, pypi, version,
|
|
996
|
+
progress=lambda done, total: typer.echo(f"rebuilt {done}/{total}", err=True),
|
|
997
|
+
) # fmt: skip
|
|
998
|
+
except update_db.NoLedgerError as error:
|
|
999
|
+
raise typer.BadParameter(str(error)) from None
|
|
1000
|
+
finally:
|
|
1001
|
+
prior.close()
|
|
1002
|
+
github.close()
|
|
1003
|
+
pypi.close()
|
|
1004
|
+
merged: dict[str, int] = {}
|
|
1005
|
+
if llm is not None:
|
|
1006
|
+
fresh = set(report.to_extract)
|
|
1007
|
+
primaries = llm_records.load(root / llm)
|
|
1008
|
+
secondaries = llm_records.load(root / second) if second is not None else {}
|
|
1009
|
+
upgraded, merged = llm_records.apply(
|
|
1010
|
+
[r for r in records if r.vulnerability in fresh], primaries, secondaries
|
|
1011
|
+
)
|
|
1012
|
+
records = [r for r in records if r.vulnerability not in fresh] + upgraded
|
|
1013
|
+
commit = subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
1014
|
+
["git", "-C", str(root), "rev-parse", "HEAD"], # noqa: S607
|
|
1015
|
+
capture_output=True, text=True, check=False,
|
|
1016
|
+
).stdout.strip() # fmt: skip
|
|
1017
|
+
meta = {
|
|
1018
|
+
"db_version": version,
|
|
1019
|
+
"built_at": datetime.now(UTC).isoformat(),
|
|
1020
|
+
"builder": prior_meta.get("builder", "fallback-v1") if llm is None else "llm-v1",
|
|
1021
|
+
"snapshot_sha256": dump_entry.sha256,
|
|
1022
|
+
"code_commit": commit,
|
|
1023
|
+
"updated_from": prior_meta.get("db_version", "?"),
|
|
1024
|
+
}
|
|
1025
|
+
target = root / out
|
|
1026
|
+
db.write(target.with_name(target.name + ".sqlite"), records, aliases, meta, builds=builds)
|
|
1027
|
+
target.with_name(target.name + "-update.json").write_text(
|
|
1028
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
1029
|
+
)
|
|
1030
|
+
typer.echo(
|
|
1031
|
+
f"db {version} from {report.previous_version}: carried {report.carried}, "
|
|
1032
|
+
f"rebuilt {report.rebuilt}, new {report.new}, removed {report.removed}; "
|
|
1033
|
+
f"{len(records)} records; to extract {len(report.to_extract)}; {report.status}"
|
|
1034
|
+
)
|
|
1035
|
+
if merged:
|
|
1036
|
+
typer.echo(f"LLM merge: {merged}")
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
@app.command("gate-db")
|
|
1040
|
+
def gate_db_command(
|
|
1041
|
+
candidate: Annotated[Path, typer.Argument(help="The candidate database.")],
|
|
1042
|
+
previous: Annotated[Path, typer.Option(help="The previous release's database.")],
|
|
1043
|
+
root: Root = Path(),
|
|
1044
|
+
out: Annotated[Path, typer.Option(help="Report path, without extension.")] = Path(
|
|
1045
|
+
"data/build/gate"
|
|
1046
|
+
),
|
|
1047
|
+
) -> None:
|
|
1048
|
+
"""Run the release gate on a candidate database; exit 1 if any check fails."""
|
|
1049
|
+
from patchsight.pipeline import gate
|
|
1050
|
+
|
|
1051
|
+
new, old = db.SymbolDatabase(root / candidate), db.SymbolDatabase(root / previous)
|
|
1052
|
+
try:
|
|
1053
|
+
report = gate.run(new, old)
|
|
1054
|
+
finally:
|
|
1055
|
+
new.close()
|
|
1056
|
+
old.close()
|
|
1057
|
+
target = root / out
|
|
1058
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1059
|
+
markdown = gate.render_markdown(report)
|
|
1060
|
+
target.with_name(target.name + ".md").write_text(markdown, "utf-8")
|
|
1061
|
+
target.with_name(target.name + ".json").write_text(
|
|
1062
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
1063
|
+
)
|
|
1064
|
+
typer.echo(markdown)
|
|
1065
|
+
if not report.passed:
|
|
1066
|
+
raise typer.Exit(code=1)
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
ATTRIBUTION = """## Attribution and licence
|
|
1070
|
+
|
|
1071
|
+
Advisory identifiers and affected-package data come from [OSV.dev](https://osv.dev), which \
|
|
1072
|
+
aggregates the [GitHub Advisory Database](https://github.com/github/advisory-database) and the \
|
|
1073
|
+
[PyPA Advisory Database](https://github.com/pypa/advisory-database). All three are licensed \
|
|
1074
|
+
CC BY 4.0.
|
|
1075
|
+
|
|
1076
|
+
**Changes made:** PatchSight resolved each advisory's fix commit on GitHub, mapped the changed \
|
|
1077
|
+
lines to function names with tree-sitter, and, where shown, had language models choose the \
|
|
1078
|
+
vulnerable ones. It stores those names with file paths, line ranges and commit SHAs. No \
|
|
1079
|
+
third-party source code is included.
|
|
1080
|
+
"""
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
@app.command("release-notes")
|
|
1084
|
+
def release_notes_command(
|
|
1085
|
+
database: Annotated[Path, typer.Argument(help="The database being released.")],
|
|
1086
|
+
pointer: Annotated[Path, typer.Option(help="Its pointer (package-db output).")] = Path(
|
|
1087
|
+
"releases/latest.json"
|
|
1088
|
+
),
|
|
1089
|
+
update: Annotated[Path | None, typer.Option(help="The update-db report, if any.")] = None,
|
|
1090
|
+
gate: Annotated[Path | None, typer.Option(help="The gate-db report, if any.")] = None,
|
|
1091
|
+
root: Root = Path(),
|
|
1092
|
+
out: Annotated[Path, typer.Option(help="Notes file to write.")] = Path(
|
|
1093
|
+
"data/build/release-notes.md"
|
|
1094
|
+
),
|
|
1095
|
+
) -> None:
|
|
1096
|
+
"""Write release notes for a database release, with provenance and attribution."""
|
|
1097
|
+
from patchsight.pipeline.gate import GateReport
|
|
1098
|
+
from patchsight.pipeline.update_db import UpdateReport
|
|
1099
|
+
|
|
1100
|
+
opened = db.SymbolDatabase(root / database)
|
|
1101
|
+
try:
|
|
1102
|
+
meta, tiers, total = dict(opened.meta), opened.confidence_counts(), opened.record_count()
|
|
1103
|
+
vulnerabilities = len(opened.vulnerabilities())
|
|
1104
|
+
finally:
|
|
1105
|
+
opened.close()
|
|
1106
|
+
published = dbcache.Pointer.model_validate_json((root / pointer).read_text("utf-8"))
|
|
1107
|
+
version = meta.get("db_version", "?")
|
|
1108
|
+
origin = f", updated from {meta['updated_from']}" if "updated_from" in meta else ""
|
|
1109
|
+
lines = [
|
|
1110
|
+
f"Symbol database **{version}**{origin}.",
|
|
1111
|
+
"",
|
|
1112
|
+
"```bash",
|
|
1113
|
+
"patchsight db update # downloads this release and verifies its SHA-256",
|
|
1114
|
+
"```",
|
|
1115
|
+
"",
|
|
1116
|
+
"## Contents",
|
|
1117
|
+
"",
|
|
1118
|
+
f"- **{vulnerabilities:,} PyPI vulnerabilities**, {total:,} records: "
|
|
1119
|
+
+ ", ".join(f"{n:,} {tier} confidence" for tier, n in sorted(tiers.items())),
|
|
1120
|
+
"- Low-confidence records list every definition the fix changed; they can mark an "
|
|
1121
|
+
"alert CALLED, never IMPORTED, NOT CALLED.",
|
|
1122
|
+
]
|
|
1123
|
+
if update is not None:
|
|
1124
|
+
report = UpdateReport.model_validate_json((root / update).read_text("utf-8"))
|
|
1125
|
+
lines += [
|
|
1126
|
+
f"- **Changes since {report.previous_version}:** {report.carried:,} carried "
|
|
1127
|
+
f"unchanged, {report.rebuilt} rebuilt, {report.new} new, {report.removed} removed"
|
|
1128
|
+
+ (f"; {report.aliases_added} aliases added" if report.aliases_added else "")
|
|
1129
|
+
+ (f", {report.aliases_removed} removed" if report.aliases_removed else ""),
|
|
1130
|
+
]
|
|
1131
|
+
if gate is not None:
|
|
1132
|
+
result = GateReport.model_validate_json((root / gate).read_text("utf-8"))
|
|
1133
|
+
lines += [
|
|
1134
|
+
"",
|
|
1135
|
+
f"## Release gate: {'passed' if result.passed else 'FAILED'}",
|
|
1136
|
+
"",
|
|
1137
|
+
*(f"- `{c.name}`: {c.detail}" for c in result.checks),
|
|
1138
|
+
]
|
|
1139
|
+
lines += [
|
|
1140
|
+
"",
|
|
1141
|
+
"## Provenance",
|
|
1142
|
+
"",
|
|
1143
|
+
f"- OSV PyPI dump SHA-256: `{meta.get('snapshot_sha256', '?')}`",
|
|
1144
|
+
f"- Built by PatchSight commit `{meta.get('code_commit', '?')}` "
|
|
1145
|
+
f"(builder `{meta.get('builder', '?')}`, schema version {meta.get('schema_version', '?')})",
|
|
1146
|
+
f"- Asset SHA-256: `{published.sha256}`",
|
|
1147
|
+
"",
|
|
1148
|
+
ATTRIBUTION,
|
|
1149
|
+
]
|
|
1150
|
+
target = root / out
|
|
1151
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1152
|
+
target.write_text("\n".join(lines), "utf-8")
|
|
1153
|
+
typer.echo(f"wrote {target}")
|
|
1154
|
+
|
|
1155
|
+
|
|
869
1156
|
@app.command("pypi-extract")
|
|
870
1157
|
def pypi_extract_command(
|
|
871
1158
|
root: Root = Path(),
|
|
@@ -899,6 +1186,12 @@ def pypi_extract_command(
|
|
|
899
1186
|
"--priority", help="Most-downloaded packages first (needs fetch-top-packages)."
|
|
900
1187
|
),
|
|
901
1188
|
] = False,
|
|
1189
|
+
cache_only: Annotated[
|
|
1190
|
+
bool,
|
|
1191
|
+
typer.Option(
|
|
1192
|
+
"--cache-only", help="Answer only from the response cache; never call a model."
|
|
1193
|
+
),
|
|
1194
|
+
] = False,
|
|
902
1195
|
) -> None:
|
|
903
1196
|
"""Run LLM extraction on PyPI vulnerabilities (needs the model's key, and GITHUB_TOKEN
|
|
904
1197
|
unless --offline)."""
|
|
@@ -954,7 +1247,8 @@ def pypi_extract_command(
|
|
|
954
1247
|
f"{sum(p.truncated for p in sized)}"
|
|
955
1248
|
)
|
|
956
1249
|
return
|
|
957
|
-
|
|
1250
|
+
make_client = cache_only_client if cache_only else client_for
|
|
1251
|
+
llm, model_id = make_client(llm_config, role, root / cache / "llm", counter)
|
|
958
1252
|
try:
|
|
959
1253
|
results, summary = pypi_extract.run(
|
|
960
1254
|
groups, by_id, github, pypi, llm, llm_config, role, template, counter,
|