patchsight 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchsight-0.1.0 → patchsight-0.3.0}/PKG-INFO +6 -5
- {patchsight-0.1.0 → patchsight-0.3.0}/docs/pypi.md +5 -4
- {patchsight-0.1.0 → patchsight-0.3.0}/pyproject.toml +1 -1
- {patchsight-0.1.0 → patchsight-0.3.0}/pyproject.toml.orig +1 -1
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/cli.py +45 -6
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/db.py +179 -1
- patchsight-0.3.0/src/patchsight/eval/failures.py +118 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/__main__.py +556 -22
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/extraction.py +9 -3
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/replay.py +5 -3
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/build_db.py +82 -36
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/github.py +2 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/extractor.py +40 -1
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/prompt.py +53 -10
- patchsight-0.3.0/src/patchsight/pipeline/gate.py +153 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/http.py +14 -1
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/snapshot.py +3 -0
- patchsight-0.3.0/src/patchsight/pipeline/kaggle_inputs.py +141 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/client.py +18 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/config.py +11 -0
- patchsight-0.3.0/src/patchsight/pipeline/llm_records.py +133 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/pypi.py +2 -0
- patchsight-0.3.0/src/patchsight/pipeline/pypi_extract.py +311 -0
- patchsight-0.3.0/src/patchsight/pipeline/update_db.py +124 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/dbcache.py +19 -9
- patchsight-0.3.0/src/patchsight/scan/explain.py +115 -0
- patchsight-0.3.0/src/patchsight/scan/lockfiles/__init__.py +47 -0
- patchsight-0.3.0/src/patchsight/scan/lockfiles/poetry.py +26 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/lockfiles/requirements.py +2 -11
- patchsight-0.3.0/src/patchsight/scan/lockfiles/toml_lock.py +41 -0
- patchsight-0.3.0/src/patchsight/scan/lockfiles/uv.py +28 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/report.py +1 -1
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/run.py +3 -4
- patchsight-0.1.0/src/patchsight/scan/lockfiles/__init__.py +0 -1
- {patchsight-0.1.0 → patchsight-0.3.0}/LICENSE +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/eval/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/eval/confidence.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/eval/metrics.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/eval/parity.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/eval/safety.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/names.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/bundle.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/changedfuncs.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/dunlap.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/benchmarks/splits.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/commits.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/hunks.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/mapping.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/diffs/paths.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/schema.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/extract/validate.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/fix_sample.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/aliases.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/ghsa_repo.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/osv.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/ingest/references.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/llm/tokens.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/pipeline/week1.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/py.typed +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/__init__.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/ast_scanner.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/osv_client.py +0 -0
- {patchsight-0.1.0 → patchsight-0.3.0}/src/patchsight/scan/verdicts.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchsight
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Know which vulnerable Python dependencies your code actually calls: symbol-level triage, no LLM at scan time.
|
|
5
5
|
Keywords: security,vulnerability,dependencies,supply-chain,sca,static-analysis,reachability,osv,pypi,cli
|
|
6
6
|
Author: Niroj33
|
|
@@ -55,15 +55,16 @@ An LLM reads each PyPI security fix offline and records which functions carry th
|
|
|
55
55
|
pip install patchsight
|
|
56
56
|
|
|
57
57
|
patchsight db update # download the latest symbol database (SHA-256 verified)
|
|
58
|
-
patchsight scan . # scan
|
|
58
|
+
patchsight scan . # scan the lockfile and the code under .
|
|
59
59
|
```
|
|
60
60
|
|
|
61
61
|
| Option | What it does |
|
|
62
62
|
|:---|:---|
|
|
63
|
-
| `--
|
|
63
|
+
| `--lockfile PATH` | Scan a specific lockfile: `requirements.txt`, `uv.lock` or `poetry.lock` (default: the first found, in that order) |
|
|
64
64
|
| `--format json` | Machine-readable output |
|
|
65
65
|
| `--fail-on called` | Exit 1 if any alert is CALLED: a CI gate (`unknown` also fails on UNKNOWN) |
|
|
66
66
|
| `--db PATH` | Use a specific database file (or set `PATCHSIGHT_DB`) |
|
|
67
|
+
| `patchsight explain ID` | Why the database lists an advisory's symbols: confidence, models, rationale and the changed lines |
|
|
67
68
|
| `patchsight db info` | Show which database a scan uses, and its age |
|
|
68
69
|
|
|
69
70
|
## Verdicts
|
|
@@ -91,8 +92,8 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
91
92
|
|
|
92
93
|
## Status: early release
|
|
93
94
|
|
|
94
|
-
-
|
|
95
|
-
- The
|
|
95
|
+
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
96
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. Expect many UNKNOWN verdicts until high-confidence records are published.
|
|
96
97
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
97
98
|
|
|
98
99
|
## Links
|
|
@@ -17,15 +17,16 @@ An LLM reads each PyPI security fix offline and records which functions carry th
|
|
|
17
17
|
pip install patchsight
|
|
18
18
|
|
|
19
19
|
patchsight db update # download the latest symbol database (SHA-256 verified)
|
|
20
|
-
patchsight scan . # scan
|
|
20
|
+
patchsight scan . # scan the lockfile and the code under .
|
|
21
21
|
```
|
|
22
22
|
|
|
23
23
|
| Option | What it does |
|
|
24
24
|
|:---|:---|
|
|
25
|
-
| `--
|
|
25
|
+
| `--lockfile PATH` | Scan a specific lockfile: `requirements.txt`, `uv.lock` or `poetry.lock` (default: the first found, in that order) |
|
|
26
26
|
| `--format json` | Machine-readable output |
|
|
27
27
|
| `--fail-on called` | Exit 1 if any alert is CALLED: a CI gate (`unknown` also fails on UNKNOWN) |
|
|
28
28
|
| `--db PATH` | Use a specific database file (or set `PATCHSIGHT_DB`) |
|
|
29
|
+
| `patchsight explain ID` | Why the database lists an advisory's symbols: confidence, models, rationale and the changed lines |
|
|
29
30
|
| `patchsight db info` | Show which database a scan uses, and its age |
|
|
30
31
|
|
|
31
32
|
## Verdicts
|
|
@@ -53,8 +54,8 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
53
54
|
|
|
54
55
|
## Status: early release
|
|
55
56
|
|
|
56
|
-
-
|
|
57
|
-
- The
|
|
57
|
+
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
58
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. Expect many UNKNOWN verdicts until high-confidence records are published.
|
|
58
59
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
59
60
|
|
|
60
61
|
## Links
|
|
@@ -89,8 +89,14 @@ def db_info(
|
|
|
89
89
|
@app.command()
|
|
90
90
|
def scan(
|
|
91
91
|
path: Annotated[Path, typer.Argument(help="Project directory to scan.")] = Path(),
|
|
92
|
-
|
|
93
|
-
Path | None,
|
|
92
|
+
lockfile_option: Annotated[
|
|
93
|
+
Path | None,
|
|
94
|
+
typer.Option(
|
|
95
|
+
"--lockfile",
|
|
96
|
+
"--requirements",
|
|
97
|
+
help="requirements.txt, uv.lock or poetry.lock (default: the first found, in "
|
|
98
|
+
"that order).",
|
|
99
|
+
),
|
|
94
100
|
] = None,
|
|
95
101
|
database: Annotated[
|
|
96
102
|
Path | None,
|
|
@@ -104,20 +110,22 @@ def scan(
|
|
|
104
110
|
"""Label every dependency alert as called, imported but not called, not imported, or unknown."""
|
|
105
111
|
from patchsight.db import DatabaseError
|
|
106
112
|
from patchsight.scan.dbcache import default_db_path
|
|
113
|
+
from patchsight.scan.lockfiles import DISCOVERY_ORDER, find
|
|
107
114
|
from patchsight.scan.osv_client import OsvError
|
|
108
115
|
from patchsight.scan.report import render_json, render_text
|
|
109
116
|
from patchsight.scan.run import run_scan
|
|
110
117
|
|
|
111
|
-
lockfile =
|
|
112
|
-
if not lockfile.
|
|
113
|
-
|
|
118
|
+
lockfile = lockfile_option or find(path)
|
|
119
|
+
if lockfile is None or not lockfile.is_file():
|
|
120
|
+
looked = lockfile if lockfile is not None else f"{path} ({', '.join(DISCOVERY_ORDER)})"
|
|
121
|
+
typer.echo(f"error: no lockfile at {looked}; pass --lockfile", err=True)
|
|
114
122
|
raise typer.Exit(code=2)
|
|
115
123
|
try:
|
|
116
124
|
alerts, dependencies, meta = run_scan(path, lockfile, database or default_db_path())
|
|
117
125
|
except DatabaseError as error:
|
|
118
126
|
typer.echo(f"error: {error}; run 'patchsight db update' or pass --db", err=True)
|
|
119
127
|
raise typer.Exit(code=2) from None
|
|
120
|
-
except OsvError as error:
|
|
128
|
+
except (OsvError, ValueError) as error:
|
|
121
129
|
typer.echo(f"error: {error}", err=True)
|
|
122
130
|
raise typer.Exit(code=2) from None
|
|
123
131
|
shown = lockfile.as_posix()
|
|
@@ -128,3 +136,34 @@ def scan(
|
|
|
128
136
|
failing = {"called": {"CALLED"}, "unknown": {"CALLED", "UNKNOWN"}}.get(fail_on, set())
|
|
129
137
|
if any(a.verdict in failing for a in alerts):
|
|
130
138
|
raise typer.Exit(code=1)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@app.command()
|
|
142
|
+
def explain(
|
|
143
|
+
identifier: Annotated[str, typer.Argument(help="Advisory ID: GHSA, PYSEC or CVE.")],
|
|
144
|
+
database: Annotated[
|
|
145
|
+
Path | None,
|
|
146
|
+
typer.Option("--db", help="Symbol database (default: the one 'db update' downloaded)."),
|
|
147
|
+
] = None,
|
|
148
|
+
output: Annotated[str, typer.Option("--format", help="text or json.")] = "text",
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Show why the database lists the symbols it does for an advisory, with evidence."""
|
|
151
|
+
from patchsight.db import DatabaseError, SymbolDatabase
|
|
152
|
+
from patchsight.scan import explain as explaining
|
|
153
|
+
from patchsight.scan.dbcache import default_db_path
|
|
154
|
+
|
|
155
|
+
try:
|
|
156
|
+
opened = SymbolDatabase(database or default_db_path())
|
|
157
|
+
except DatabaseError as error:
|
|
158
|
+
typer.echo(f"error: {error}; run 'patchsight db update' or pass --db", err=True)
|
|
159
|
+
raise typer.Exit(code=2) from None
|
|
160
|
+
try:
|
|
161
|
+
vulnerability, identifiers, records = explaining.lookup(opened, identifier)
|
|
162
|
+
meta = opened.meta
|
|
163
|
+
finally:
|
|
164
|
+
opened.close()
|
|
165
|
+
canonical = explaining.normalise_identifier(identifier)
|
|
166
|
+
render = explaining.render_json if output == "json" else explaining.render_text
|
|
167
|
+
typer.echo(render(canonical, vulnerability, identifiers, records, meta))
|
|
168
|
+
if not records:
|
|
169
|
+
raise typer.Exit(code=1)
|
|
@@ -9,6 +9,13 @@ its vulnerability, so the scanner can look up whatever ID OSV returns.
|
|
|
9
9
|
Low-confidence symbols can make an alert ``CALLED`` but never ``IMPORTED_NOT_CALLED``.
|
|
10
10
|
|
|
11
11
|
Stdlib ``sqlite3`` only, so the scan path can read it without any pipeline dependency.
|
|
12
|
+
|
|
13
|
+
Compatibility: three tables were added within schema version 1. ``extraction`` holds the
|
|
14
|
+
second-opinion model and the reason for the tier; ``evidence`` holds each flagged symbol's
|
|
15
|
+
rationale and cited hunk locations; ``builds`` is the ledger of every vulnerability
|
|
16
|
+
considered, with its input hash, for incremental updates. Readers that predate them never
|
|
17
|
+
query them, so they keep working; this reader treats a database without them as having no
|
|
18
|
+
such data. Evidence holds commit SHAs, paths and line ranges only, never code (rule 13).
|
|
12
19
|
"""
|
|
13
20
|
|
|
14
21
|
import json
|
|
@@ -55,6 +62,26 @@ CREATE TABLE symbols (
|
|
|
55
62
|
FOREIGN KEY (vulnerability, package) REFERENCES records (vulnerability, package)
|
|
56
63
|
);
|
|
57
64
|
CREATE INDEX symbols_by_record ON symbols (vulnerability, package);
|
|
65
|
+
CREATE TABLE extraction (
|
|
66
|
+
vulnerability TEXT NOT NULL,
|
|
67
|
+
package TEXT NOT NULL,
|
|
68
|
+
second_opinion_model TEXT,
|
|
69
|
+
tier_reason TEXT,
|
|
70
|
+
PRIMARY KEY (vulnerability, package)
|
|
71
|
+
);
|
|
72
|
+
CREATE TABLE builds (
|
|
73
|
+
vulnerability TEXT PRIMARY KEY,
|
|
74
|
+
input_hash TEXT NOT NULL,
|
|
75
|
+
status TEXT NOT NULL
|
|
76
|
+
);
|
|
77
|
+
CREATE TABLE evidence (
|
|
78
|
+
vulnerability TEXT NOT NULL,
|
|
79
|
+
package TEXT NOT NULL,
|
|
80
|
+
qualified_name TEXT NOT NULL,
|
|
81
|
+
rationale TEXT NOT NULL,
|
|
82
|
+
hunks TEXT NOT NULL,
|
|
83
|
+
PRIMARY KEY (vulnerability, package, qualified_name)
|
|
84
|
+
);
|
|
58
85
|
"""
|
|
59
86
|
|
|
60
87
|
|
|
@@ -69,6 +96,40 @@ class FixCommitRef(BaseModel):
|
|
|
69
96
|
sha: str
|
|
70
97
|
|
|
71
98
|
|
|
99
|
+
class BuildEntry(BaseModel):
|
|
100
|
+
"""One vulnerability the builder considered, with or without a record: the ledger an
|
|
101
|
+
incremental update compares against (ADR 0001: re-extract only when inputs change)."""
|
|
102
|
+
|
|
103
|
+
model_config = ConfigDict(frozen=True)
|
|
104
|
+
|
|
105
|
+
vulnerability: str
|
|
106
|
+
input_hash: str
|
|
107
|
+
status: str
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class HunkLocation(BaseModel):
|
|
111
|
+
"""Where a cited diff hunk sits: the fix commit, the file and both line ranges."""
|
|
112
|
+
|
|
113
|
+
model_config = ConfigDict(frozen=True)
|
|
114
|
+
|
|
115
|
+
commit: str
|
|
116
|
+
path: str
|
|
117
|
+
old_start: int
|
|
118
|
+
old_count: int
|
|
119
|
+
new_start: int
|
|
120
|
+
new_count: int
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class SymbolEvidence(BaseModel):
|
|
124
|
+
"""Why the model flagged a symbol: a one-sentence rationale and the hunks it cited."""
|
|
125
|
+
|
|
126
|
+
model_config = ConfigDict(frozen=True)
|
|
127
|
+
|
|
128
|
+
qualified_name: str
|
|
129
|
+
rationale: str
|
|
130
|
+
hunks: tuple[HunkLocation, ...]
|
|
131
|
+
|
|
132
|
+
|
|
72
133
|
class Symbol(BaseModel):
|
|
73
134
|
"""A vulnerable definition. ``qualified_name`` is ``module.qualname`` when the module
|
|
74
135
|
is known, otherwise just the qualname."""
|
|
@@ -99,10 +160,20 @@ class Record(BaseModel):
|
|
|
99
160
|
advisory_modified: datetime
|
|
100
161
|
truncated: bool
|
|
101
162
|
extracted_at: datetime
|
|
163
|
+
# LLM records only: the second model's ID (ADR 0005) and why the tier was chosen.
|
|
164
|
+
second_opinion_model: str | None = None
|
|
165
|
+
tier_reason: str | None = None
|
|
166
|
+
# The symbols the model judged vulnerable (all of a high record's symbols; in a low
|
|
167
|
+
# record, the model's picks among every changed definition).
|
|
168
|
+
evidence: tuple[SymbolEvidence, ...] = ()
|
|
102
169
|
|
|
103
170
|
|
|
104
171
|
def write(
|
|
105
|
-
path: Path,
|
|
172
|
+
path: Path,
|
|
173
|
+
records: Iterable[Record],
|
|
174
|
+
aliases: dict[str, str],
|
|
175
|
+
meta: dict[str, str],
|
|
176
|
+
builds: Iterable[BuildEntry] = (),
|
|
106
177
|
) -> int:
|
|
107
178
|
"""Write a fresh database file. Returns the number of records written."""
|
|
108
179
|
if path.exists():
|
|
@@ -117,6 +188,10 @@ def write(
|
|
|
117
188
|
sorted({**meta, "schema_version": str(SCHEMA_VERSION)}.items()),
|
|
118
189
|
)
|
|
119
190
|
connection.executemany("INSERT INTO aliases VALUES (?, ?)", sorted(aliases.items()))
|
|
191
|
+
connection.executemany(
|
|
192
|
+
"INSERT INTO builds VALUES (?, ?, ?)",
|
|
193
|
+
sorted((b.vulnerability, b.input_hash, b.status) for b in builds),
|
|
194
|
+
)
|
|
120
195
|
for record in records:
|
|
121
196
|
count += 1
|
|
122
197
|
connection.execute(
|
|
@@ -137,6 +212,29 @@ def write(
|
|
|
137
212
|
record.extracted_at.isoformat(),
|
|
138
213
|
),
|
|
139
214
|
)
|
|
215
|
+
if record.second_opinion_model is not None or record.tier_reason is not None:
|
|
216
|
+
connection.execute(
|
|
217
|
+
"INSERT INTO extraction VALUES (?, ?, ?, ?)",
|
|
218
|
+
(
|
|
219
|
+
record.vulnerability,
|
|
220
|
+
record.package,
|
|
221
|
+
record.second_opinion_model,
|
|
222
|
+
record.tier_reason,
|
|
223
|
+
),
|
|
224
|
+
)
|
|
225
|
+
connection.executemany(
|
|
226
|
+
"INSERT INTO evidence VALUES (?, ?, ?, ?, ?)",
|
|
227
|
+
[
|
|
228
|
+
(
|
|
229
|
+
record.vulnerability,
|
|
230
|
+
record.package,
|
|
231
|
+
e.qualified_name,
|
|
232
|
+
e.rationale,
|
|
233
|
+
json.dumps([h.model_dump() for h in e.hunks]),
|
|
234
|
+
)
|
|
235
|
+
for e in record.evidence
|
|
236
|
+
],
|
|
237
|
+
)
|
|
140
238
|
connection.executemany(
|
|
141
239
|
"INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
142
240
|
[
|
|
@@ -175,6 +273,15 @@ class SymbolDatabase:
|
|
|
175
273
|
f"{path} has schema version {version}; this version of PatchSight reads "
|
|
176
274
|
f"version {SCHEMA_VERSION}. Update PatchSight or download a matching database."
|
|
177
275
|
)
|
|
276
|
+
tables = {
|
|
277
|
+
name
|
|
278
|
+
for (name,) in self._connection.execute(
|
|
279
|
+
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
|
280
|
+
)
|
|
281
|
+
}
|
|
282
|
+
self._has_extraction = "extraction" in tables
|
|
283
|
+
self._has_evidence = "evidence" in tables
|
|
284
|
+
self._has_builds = "builds" in tables
|
|
178
285
|
|
|
179
286
|
def close(self) -> None:
|
|
180
287
|
self._connection.close()
|
|
@@ -188,6 +295,49 @@ class SymbolDatabase:
|
|
|
188
295
|
).fetchone()
|
|
189
296
|
return str(row[0]) if row else None
|
|
190
297
|
|
|
298
|
+
def confidence_counts(self) -> dict[str, int]:
|
|
299
|
+
"""Records by confidence tier."""
|
|
300
|
+
rows = self._connection.execute(
|
|
301
|
+
"SELECT confidence, COUNT(*) FROM records GROUP BY confidence ORDER BY confidence"
|
|
302
|
+
)
|
|
303
|
+
return {str(r[0]): int(r[1]) for r in rows}
|
|
304
|
+
|
|
305
|
+
def vulnerabilities(self) -> list[str]:
|
|
306
|
+
"""Every vulnerability that has at least one record."""
|
|
307
|
+
rows = self._connection.execute(
|
|
308
|
+
"SELECT DISTINCT vulnerability FROM records ORDER BY vulnerability"
|
|
309
|
+
)
|
|
310
|
+
return [str(r[0]) for r in rows]
|
|
311
|
+
|
|
312
|
+
def alias_map(self) -> dict[str, str]:
|
|
313
|
+
"""Every advisory ID and alias, mapped to its vulnerability."""
|
|
314
|
+
rows = self._connection.execute("SELECT identifier, vulnerability FROM aliases")
|
|
315
|
+
return {str(r[0]): str(r[1]) for r in rows}
|
|
316
|
+
|
|
317
|
+
def build_ledger(self) -> dict[str, BuildEntry]:
|
|
318
|
+
"""Every vulnerability the builder considered; empty for an older database."""
|
|
319
|
+
if not self._has_builds:
|
|
320
|
+
return {}
|
|
321
|
+
rows = self._connection.execute("SELECT vulnerability, input_hash, status FROM builds")
|
|
322
|
+
return {r[0]: BuildEntry(vulnerability=r[0], input_hash=r[1], status=r[2]) for r in rows}
|
|
323
|
+
|
|
324
|
+
def identifiers_for(self, vulnerability: str) -> list[str]:
|
|
325
|
+
"""Every advisory ID and alias that points at this vulnerability."""
|
|
326
|
+
rows = self._connection.execute(
|
|
327
|
+
"SELECT identifier FROM aliases WHERE vulnerability = ? ORDER BY identifier",
|
|
328
|
+
(vulnerability,),
|
|
329
|
+
).fetchall()
|
|
330
|
+
return [str(r[0]) for r in rows]
|
|
331
|
+
|
|
332
|
+
def records_for(self, vulnerability: str) -> list[Record]:
|
|
333
|
+
"""The vulnerability's records, one per affected package."""
|
|
334
|
+
packages = self._connection.execute(
|
|
335
|
+
"SELECT package FROM records WHERE vulnerability = ? ORDER BY package",
|
|
336
|
+
(vulnerability,),
|
|
337
|
+
).fetchall()
|
|
338
|
+
found = (self.record(vulnerability, str(p[0])) for p in packages)
|
|
339
|
+
return [r for r in found if r is not None]
|
|
340
|
+
|
|
191
341
|
def record(self, vulnerability: str, package: str) -> Record | None:
|
|
192
342
|
row = self._connection.execute(
|
|
193
343
|
"SELECT * FROM records WHERE vulnerability = ? AND package = ?",
|
|
@@ -200,6 +350,24 @@ class SymbolDatabase:
|
|
|
200
350
|
"WHERE vulnerability = ? AND package = ? ORDER BY qualified_name",
|
|
201
351
|
(vulnerability, package),
|
|
202
352
|
).fetchall()
|
|
353
|
+
provenance = (
|
|
354
|
+
self._connection.execute(
|
|
355
|
+
"SELECT second_opinion_model, tier_reason FROM extraction "
|
|
356
|
+
"WHERE vulnerability = ? AND package = ?",
|
|
357
|
+
(vulnerability, package),
|
|
358
|
+
).fetchone()
|
|
359
|
+
if self._has_extraction
|
|
360
|
+
else None
|
|
361
|
+
)
|
|
362
|
+
evidence = (
|
|
363
|
+
self._connection.execute(
|
|
364
|
+
"SELECT qualified_name, rationale, hunks FROM evidence "
|
|
365
|
+
"WHERE vulnerability = ? AND package = ? ORDER BY qualified_name",
|
|
366
|
+
(vulnerability, package),
|
|
367
|
+
).fetchall()
|
|
368
|
+
if self._has_evidence
|
|
369
|
+
else []
|
|
370
|
+
)
|
|
203
371
|
return Record(
|
|
204
372
|
vulnerability=row[0],
|
|
205
373
|
package=row[1],
|
|
@@ -218,4 +386,14 @@ class SymbolDatabase:
|
|
|
218
386
|
Symbol(qualified_name=s[0], module=s[1], name=s[2], kind=s[3], path=s[4])
|
|
219
387
|
for s in symbols
|
|
220
388
|
),
|
|
389
|
+
second_opinion_model=provenance[0] if provenance else None,
|
|
390
|
+
tier_reason=provenance[1] if provenance else None,
|
|
391
|
+
evidence=tuple(
|
|
392
|
+
SymbolEvidence(
|
|
393
|
+
qualified_name=e[0],
|
|
394
|
+
rationale=e[1],
|
|
395
|
+
hunks=tuple(HunkLocation(**h) for h in json.loads(e[2])),
|
|
396
|
+
)
|
|
397
|
+
for e in evidence
|
|
398
|
+
),
|
|
221
399
|
)
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Failure causes for extraction errors (proposal §10, "failure analysis").
|
|
2
|
+
|
|
3
|
+
Every missed true symbol (a false negative) gets exactly one cause, checked in this order:
|
|
4
|
+
|
|
5
|
+
* ``not_changed``: the fix did not change the symbol, so no patch-based method can find it
|
|
6
|
+
(the ceiling ChangedFuncs also hits: "vulnerable function unchanged, callee changed");
|
|
7
|
+
* ``name_format``: the fix changed a definition that matches the symbol only on a dotted
|
|
8
|
+
boundary (GoVulnDB's ``decryptTicket`` for the method ``Conn.decryptTicket``): a naming
|
|
9
|
+
difference in the ground truth, not a missed function;
|
|
10
|
+
* ``dead_letter``: the symbol was changed, but the extraction failed;
|
|
11
|
+
* ``truncated``: changed, but dropped from the prompt to fit the token budget;
|
|
12
|
+
* ``model_missed``: shown to the model and judged not vulnerable.
|
|
13
|
+
|
|
14
|
+
Every false positive is either ``name_format`` (it matches a true symbol on a dotted
|
|
15
|
+
boundary) or ``model_false_positive``.
|
|
16
|
+
|
|
17
|
+
A report-level miss (no true symbol found: the safety metric) is attributed to its most
|
|
18
|
+
actionable cause: a shown-but-missed symbol first, then truncation, then the rest.
|
|
19
|
+
|
|
20
|
+
Not separated: "the fix adds a new helper". The replay data does not record which side of
|
|
21
|
+
the diff a candidate comes from, so such cases fall under ``model_false_positive``.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from collections import Counter
|
|
25
|
+
from collections.abc import Mapping, Set
|
|
26
|
+
|
|
27
|
+
from pydantic import BaseModel, ConfigDict
|
|
28
|
+
|
|
29
|
+
from patchsight.eval.safety import lenient_match
|
|
30
|
+
|
|
31
|
+
MISS_CAUSES = ("not_changed", "name_format", "dead_letter", "truncated", "model_missed")
|
|
32
|
+
REPORT_MISS_ORDER = ("model_missed", "truncated", "dead_letter", "name_format", "not_changed")
|
|
33
|
+
VALID = frozenset({"ok", "repaired"})
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ReportInput(BaseModel):
|
|
37
|
+
model_config = ConfigDict(frozen=True)
|
|
38
|
+
|
|
39
|
+
truth: frozenset[str]
|
|
40
|
+
changed: frozenset[str] # every changed function (ChangedFuncs)
|
|
41
|
+
shown: frozenset[str] # the candidates the prompt kept
|
|
42
|
+
status: str
|
|
43
|
+
predicted: frozenset[str]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class FailureBreakdown(BaseModel):
|
|
47
|
+
model_config = ConfigDict(frozen=True)
|
|
48
|
+
|
|
49
|
+
reports: int
|
|
50
|
+
truth_symbols: int
|
|
51
|
+
false_negatives: dict[str, int]
|
|
52
|
+
false_positives: dict[str, int]
|
|
53
|
+
report_misses: dict[str, int]
|
|
54
|
+
examples: dict[str, tuple[str, ...]] # cause -> report IDs (sorted)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def miss_cause(symbol: str, report: ReportInput) -> str:
|
|
58
|
+
if symbol not in report.changed:
|
|
59
|
+
if any(lenient_match(c, symbol) for c in report.changed):
|
|
60
|
+
return "name_format"
|
|
61
|
+
return "not_changed"
|
|
62
|
+
if report.status not in VALID:
|
|
63
|
+
return "dead_letter"
|
|
64
|
+
if symbol not in report.shown:
|
|
65
|
+
return "truncated"
|
|
66
|
+
return "model_missed"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def breakdown(reports: Mapping[str, ReportInput]) -> FailureBreakdown:
|
|
70
|
+
negatives: Counter[str] = Counter()
|
|
71
|
+
positives: Counter[str] = Counter()
|
|
72
|
+
misses: Counter[str] = Counter()
|
|
73
|
+
examples: dict[str, set[str]] = {}
|
|
74
|
+
total = 0
|
|
75
|
+
for rid, report in sorted(reports.items()):
|
|
76
|
+
total += len(report.truth)
|
|
77
|
+
causes = {miss_cause(s, report) for s in report.truth - report.predicted}
|
|
78
|
+
for symbol in report.truth - report.predicted:
|
|
79
|
+
cause = miss_cause(symbol, report)
|
|
80
|
+
negatives[cause] += 1
|
|
81
|
+
examples.setdefault(cause, set()).add(rid)
|
|
82
|
+
for symbol in report.predicted - report.truth:
|
|
83
|
+
cause = (
|
|
84
|
+
"name_format"
|
|
85
|
+
if any(lenient_match(symbol, t) for t in report.truth)
|
|
86
|
+
else "model_false_positive"
|
|
87
|
+
)
|
|
88
|
+
positives[cause] += 1
|
|
89
|
+
examples.setdefault(f"fp:{cause}", set()).add(rid)
|
|
90
|
+
if report.truth and not report.truth & report.predicted:
|
|
91
|
+
misses[next(c for c in REPORT_MISS_ORDER if c in causes)] += 1
|
|
92
|
+
return FailureBreakdown(
|
|
93
|
+
reports=len(reports),
|
|
94
|
+
truth_symbols=total,
|
|
95
|
+
false_negatives={c: negatives[c] for c in MISS_CAUSES},
|
|
96
|
+
false_positives={c: positives[c] for c in ("name_format", "model_false_positive")},
|
|
97
|
+
report_misses={c: misses[c] for c in REPORT_MISS_ORDER},
|
|
98
|
+
examples={k: tuple(sorted(v)) for k, v in sorted(examples.items())},
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def as_inputs(
|
|
103
|
+
truth: Mapping[str, Set[str]],
|
|
104
|
+
changed: Mapping[str, Set[str]],
|
|
105
|
+
shown: Mapping[str, Set[str]],
|
|
106
|
+
status: Mapping[str, str],
|
|
107
|
+
predicted: Mapping[str, Set[str]],
|
|
108
|
+
) -> dict[str, ReportInput]:
|
|
109
|
+
return {
|
|
110
|
+
r: ReportInput(
|
|
111
|
+
truth=frozenset(truth[r]),
|
|
112
|
+
changed=frozenset(changed.get(r, set())),
|
|
113
|
+
shown=frozenset(shown.get(r, set())),
|
|
114
|
+
status=status.get(r, "dead_letter"),
|
|
115
|
+
predicted=frozenset(predicted.get(r, set())),
|
|
116
|
+
)
|
|
117
|
+
for r in truth
|
|
118
|
+
}
|