patchsight 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchsight-0.2.0 → patchsight-0.4.0}/PKG-INFO +12 -3
- {patchsight-0.2.0 → patchsight-0.4.0}/docs/pypi.md +11 -2
- {patchsight-0.2.0 → patchsight-0.4.0}/pyproject.toml +1 -1
- {patchsight-0.2.0 → patchsight-0.4.0}/pyproject.toml.orig +1 -1
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/cli.py +13 -1
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/db.py +62 -6
- patchsight-0.4.0/src/patchsight/eval/failures.py +118 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/__main__.py +419 -6
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/extraction.py +1 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/build_db.py +18 -5
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/extractor.py +2 -1
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/prompt.py +16 -5
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/validate.py +24 -4
- patchsight-0.4.0/src/patchsight/pipeline/gate.py +173 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/client.py +18 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/config.py +9 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm_records.py +76 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/pypi_extract.py +9 -6
- patchsight-0.4.0/src/patchsight/pipeline/update_db.py +124 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/dbcache.py +19 -9
- patchsight-0.4.0/src/patchsight/scan/sarif.py +177 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/LICENSE +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/eval/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/eval/confidence.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/eval/metrics.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/eval/parity.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/eval/safety.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/names.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/bundle.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/changedfuncs.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/dunlap.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/replay.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/splits.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/commits.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/github.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/hunks.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/mapping.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/paths.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/schema.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/fix_sample.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/http.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/aliases.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/ghsa_repo.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/osv.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/references.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/snapshot.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/kaggle_inputs.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/tokens.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/pypi.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/pipeline/week1.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/py.typed +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/ast_scanner.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/explain.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/__init__.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/poetry.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/requirements.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/toml_lock.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/uv.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/osv_client.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/report.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/run.py +0 -0
- {patchsight-0.2.0 → patchsight-0.4.0}/src/patchsight/scan/verdicts.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchsight
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Know which vulnerable Python dependencies your code actually calls: symbol-level triage, no LLM at scan time.
|
|
5
5
|
Keywords: security,vulnerability,dependencies,supply-chain,sca,static-analysis,reachability,osv,pypi,cli
|
|
6
6
|
Author: Niroj33
|
|
@@ -62,17 +62,26 @@ patchsight scan . # scan the lockfile and the code under .
|
|
|
62
62
|
|:---|:---|
|
|
63
63
|
| `--lockfile PATH` | Scan a specific lockfile: `requirements.txt`, `uv.lock` or `poetry.lock` (default: the first found, in that order) |
|
|
64
64
|
| `--format json` | Machine-readable output |
|
|
65
|
+
| `--format sarif` | SARIF 2.1.0 for GitHub code scanning and other SARIF readers |
|
|
65
66
|
| `--fail-on called` | Exit 1 if any alert is CALLED: a CI gate (`unknown` also fails on UNKNOWN) |
|
|
66
67
|
| `--db PATH` | Use a specific database file (or set `PATCHSIGHT_DB`) |
|
|
67
68
|
| `patchsight explain ID` | Why the database lists an advisory's symbols: confidence, models, rationale and the changed lines |
|
|
68
69
|
| `patchsight db info` | Show which database a scan uses, and its age |
|
|
69
70
|
|
|
71
|
+
In GitHub Actions, the [PatchSight action](https://github.com/professor3333/PatchSight#in-ci-the-github-action) scans and uploads the SARIF report to code scanning:
|
|
72
|
+
|
|
73
|
+
```yaml
|
|
74
|
+
- uses: professor3333/PatchSight@v0.4.0
|
|
75
|
+
with:
|
|
76
|
+
fail-on: called
|
|
77
|
+
```
|
|
78
|
+
|
|
70
79
|
## Verdicts
|
|
71
80
|
|
|
72
81
|
| | Verdict | Meaning |
|
|
73
82
|
|:---:|:---|:---|
|
|
74
83
|
| 🔴 | **CALLED** | A reference in your code resolves to a vulnerable function, its class, or an entry point that reaches it. **Fix these first.** |
|
|
75
|
-
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data. |
|
|
84
|
+
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data, which is [withheld for now](https://github.com/professor3333/PatchSight/blob/main/docs/adr/0008-withhold-high-confidence.md). |
|
|
76
85
|
| ⚪ | **NOT IMPORTED** | Your code never imports the package directly. It may still be used through other dependencies. |
|
|
77
86
|
| 🟡 | **UNKNOWN** | No symbol data, only low-confidence data, or dynamic code the scanner can't follow. |
|
|
78
87
|
|
|
@@ -93,7 +102,7 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
93
102
|
## Status: early release
|
|
94
103
|
|
|
95
104
|
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
96
|
-
- The
|
|
105
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. High-confidence records are withheld until the confidence rule meets its safety target, so expect UNKNOWN where a later release may say IMPORTED, NOT CALLED.
|
|
97
106
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
98
107
|
|
|
99
108
|
## Links
|
|
@@ -24,17 +24,26 @@ patchsight scan . # scan the lockfile and the code under .
|
|
|
24
24
|
|:---|:---|
|
|
25
25
|
| `--lockfile PATH` | Scan a specific lockfile: `requirements.txt`, `uv.lock` or `poetry.lock` (default: the first found, in that order) |
|
|
26
26
|
| `--format json` | Machine-readable output |
|
|
27
|
+
| `--format sarif` | SARIF 2.1.0 for GitHub code scanning and other SARIF readers |
|
|
27
28
|
| `--fail-on called` | Exit 1 if any alert is CALLED: a CI gate (`unknown` also fails on UNKNOWN) |
|
|
28
29
|
| `--db PATH` | Use a specific database file (or set `PATCHSIGHT_DB`) |
|
|
29
30
|
| `patchsight explain ID` | Why the database lists an advisory's symbols: confidence, models, rationale and the changed lines |
|
|
30
31
|
| `patchsight db info` | Show which database a scan uses, and its age |
|
|
31
32
|
|
|
33
|
+
In GitHub Actions, the [PatchSight action](https://github.com/professor3333/PatchSight#in-ci-the-github-action) scans and uploads the SARIF report to code scanning:
|
|
34
|
+
|
|
35
|
+
```yaml
|
|
36
|
+
- uses: professor3333/PatchSight@v0.4.0
|
|
37
|
+
with:
|
|
38
|
+
fail-on: called
|
|
39
|
+
```
|
|
40
|
+
|
|
32
41
|
## Verdicts
|
|
33
42
|
|
|
34
43
|
| | Verdict | Meaning |
|
|
35
44
|
|:---:|:---|:---|
|
|
36
45
|
| 🔴 | **CALLED** | A reference in your code resolves to a vulnerable function, its class, or an entry point that reaches it. **Fix these first.** |
|
|
37
|
-
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data. |
|
|
46
|
+
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data, which is [withheld for now](https://github.com/professor3333/PatchSight/blob/main/docs/adr/0008-withhold-high-confidence.md). |
|
|
38
47
|
| ⚪ | **NOT IMPORTED** | Your code never imports the package directly. It may still be used through other dependencies. |
|
|
39
48
|
| 🟡 | **UNKNOWN** | No symbol data, only low-confidence data, or dynamic code the scanner can't follow. |
|
|
40
49
|
|
|
@@ -55,7 +64,7 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
55
64
|
## Status: early release
|
|
56
65
|
|
|
57
66
|
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
58
|
-
- The
|
|
67
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. High-confidence records are withheld until the confidence rule meets its safety target, so expect UNKNOWN where a later release may say IMPORTED, NOT CALLED.
|
|
59
68
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
60
69
|
|
|
61
70
|
## Links
|
|
@@ -102,7 +102,9 @@ def scan(
|
|
|
102
102
|
Path | None,
|
|
103
103
|
typer.Option("--db", help="Symbol database (default: the one 'db update' downloaded)."),
|
|
104
104
|
] = None,
|
|
105
|
-
output: Annotated[
|
|
105
|
+
output: Annotated[
|
|
106
|
+
str, typer.Option("--format", help="text, json, or sarif (for code scanning).")
|
|
107
|
+
] = "text",
|
|
106
108
|
fail_on: Annotated[
|
|
107
109
|
str, typer.Option(help="Exit 1 when an alert is: called, unknown (or called), or never.")
|
|
108
110
|
] = "never",
|
|
@@ -131,6 +133,16 @@ def scan(
|
|
|
131
133
|
shown = lockfile.as_posix()
|
|
132
134
|
if output == "json":
|
|
133
135
|
typer.echo(render_json(alerts, dependencies, shown, meta))
|
|
136
|
+
elif output == "sarif":
|
|
137
|
+
import os
|
|
138
|
+
|
|
139
|
+
from patchsight.scan.sarif import render_sarif
|
|
140
|
+
|
|
141
|
+
# Code locations are relative to the scanned directory; SARIF wants them relative to
|
|
142
|
+
# where the scan runs (the repository root, in CI).
|
|
143
|
+
root = path if path.is_dir() else path.parent
|
|
144
|
+
base = os.path.relpath(root.resolve(), Path.cwd().resolve())
|
|
145
|
+
typer.echo(render_sarif(alerts, dependencies, shown, meta, base))
|
|
134
146
|
else:
|
|
135
147
|
typer.echo(render_text(alerts, dependencies, shown, meta))
|
|
136
148
|
failing = {"called": {"CALLED"}, "unknown": {"CALLED", "UNKNOWN"}}.get(fail_on, set())
|
|
@@ -10,11 +10,12 @@ its vulnerability, so the scanner can look up whatever ID OSV returns.
|
|
|
10
10
|
|
|
11
11
|
Stdlib ``sqlite3`` only, so the scan path can read it without any pipeline dependency.
|
|
12
12
|
|
|
13
|
-
Compatibility:
|
|
14
|
-
and the ``evidence``
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
Compatibility: three tables were added within schema version 1. ``extraction`` holds the
|
|
14
|
+
second-opinion model and the reason for the tier; ``evidence`` holds each flagged symbol's
|
|
15
|
+
rationale and cited hunk locations; ``builds`` is the ledger of every vulnerability
|
|
16
|
+
considered, with its input hash, for incremental updates. Readers that predate them never
|
|
17
|
+
query them, so they keep working; this reader treats a database without them as having no
|
|
18
|
+
such data. Evidence holds commit SHAs, paths and line ranges only, never code (rule 13).
|
|
18
19
|
"""
|
|
19
20
|
|
|
20
21
|
import json
|
|
@@ -68,6 +69,11 @@ CREATE TABLE extraction (
|
|
|
68
69
|
tier_reason TEXT,
|
|
69
70
|
PRIMARY KEY (vulnerability, package)
|
|
70
71
|
);
|
|
72
|
+
CREATE TABLE builds (
|
|
73
|
+
vulnerability TEXT PRIMARY KEY,
|
|
74
|
+
input_hash TEXT NOT NULL,
|
|
75
|
+
status TEXT NOT NULL
|
|
76
|
+
);
|
|
71
77
|
CREATE TABLE evidence (
|
|
72
78
|
vulnerability TEXT NOT NULL,
|
|
73
79
|
package TEXT NOT NULL,
|
|
@@ -90,6 +96,17 @@ class FixCommitRef(BaseModel):
|
|
|
90
96
|
sha: str
|
|
91
97
|
|
|
92
98
|
|
|
99
|
+
class BuildEntry(BaseModel):
|
|
100
|
+
"""One vulnerability the builder considered, with or without a record: the ledger an
|
|
101
|
+
incremental update compares against (ADR 0001: re-extract only when inputs change)."""
|
|
102
|
+
|
|
103
|
+
model_config = ConfigDict(frozen=True)
|
|
104
|
+
|
|
105
|
+
vulnerability: str
|
|
106
|
+
input_hash: str
|
|
107
|
+
status: str
|
|
108
|
+
|
|
109
|
+
|
|
93
110
|
class HunkLocation(BaseModel):
|
|
94
111
|
"""Where a cited diff hunk sits: the fix commit, the file and both line ranges."""
|
|
95
112
|
|
|
@@ -152,7 +169,11 @@ class Record(BaseModel):
|
|
|
152
169
|
|
|
153
170
|
|
|
154
171
|
def write(
|
|
155
|
-
path: Path,
|
|
172
|
+
path: Path,
|
|
173
|
+
records: Iterable[Record],
|
|
174
|
+
aliases: dict[str, str],
|
|
175
|
+
meta: dict[str, str],
|
|
176
|
+
builds: Iterable[BuildEntry] = (),
|
|
156
177
|
) -> int:
|
|
157
178
|
"""Write a fresh database file. Returns the number of records written."""
|
|
158
179
|
if path.exists():
|
|
@@ -167,6 +188,10 @@ def write(
|
|
|
167
188
|
sorted({**meta, "schema_version": str(SCHEMA_VERSION)}.items()),
|
|
168
189
|
)
|
|
169
190
|
connection.executemany("INSERT INTO aliases VALUES (?, ?)", sorted(aliases.items()))
|
|
191
|
+
connection.executemany(
|
|
192
|
+
"INSERT INTO builds VALUES (?, ?, ?)",
|
|
193
|
+
sorted((b.vulnerability, b.input_hash, b.status) for b in builds),
|
|
194
|
+
)
|
|
170
195
|
for record in records:
|
|
171
196
|
count += 1
|
|
172
197
|
connection.execute(
|
|
@@ -256,6 +281,7 @@ class SymbolDatabase:
|
|
|
256
281
|
}
|
|
257
282
|
self._has_extraction = "extraction" in tables
|
|
258
283
|
self._has_evidence = "evidence" in tables
|
|
284
|
+
self._has_builds = "builds" in tables
|
|
259
285
|
|
|
260
286
|
def close(self) -> None:
|
|
261
287
|
self._connection.close()
|
|
@@ -269,6 +295,36 @@ class SymbolDatabase:
|
|
|
269
295
|
).fetchone()
|
|
270
296
|
return str(row[0]) if row else None
|
|
271
297
|
|
|
298
|
+
def all_records(self) -> list[Record]:
|
|
299
|
+
"""Every record, ordered by vulnerability and package."""
|
|
300
|
+
return [r for v in self.vulnerabilities() for r in self.records_for(v)]
|
|
301
|
+
|
|
302
|
+
def confidence_counts(self) -> dict[str, int]:
|
|
303
|
+
"""Records by confidence tier."""
|
|
304
|
+
rows = self._connection.execute(
|
|
305
|
+
"SELECT confidence, COUNT(*) FROM records GROUP BY confidence ORDER BY confidence"
|
|
306
|
+
)
|
|
307
|
+
return {str(r[0]): int(r[1]) for r in rows}
|
|
308
|
+
|
|
309
|
+
def vulnerabilities(self) -> list[str]:
|
|
310
|
+
"""Every vulnerability that has at least one record."""
|
|
311
|
+
rows = self._connection.execute(
|
|
312
|
+
"SELECT DISTINCT vulnerability FROM records ORDER BY vulnerability"
|
|
313
|
+
)
|
|
314
|
+
return [str(r[0]) for r in rows]
|
|
315
|
+
|
|
316
|
+
def alias_map(self) -> dict[str, str]:
|
|
317
|
+
"""Every advisory ID and alias, mapped to its vulnerability."""
|
|
318
|
+
rows = self._connection.execute("SELECT identifier, vulnerability FROM aliases")
|
|
319
|
+
return {str(r[0]): str(r[1]) for r in rows}
|
|
320
|
+
|
|
321
|
+
def build_ledger(self) -> dict[str, BuildEntry]:
|
|
322
|
+
"""Every vulnerability the builder considered; empty for an older database."""
|
|
323
|
+
if not self._has_builds:
|
|
324
|
+
return {}
|
|
325
|
+
rows = self._connection.execute("SELECT vulnerability, input_hash, status FROM builds")
|
|
326
|
+
return {r[0]: BuildEntry(vulnerability=r[0], input_hash=r[1], status=r[2]) for r in rows}
|
|
327
|
+
|
|
272
328
|
def identifiers_for(self, vulnerability: str) -> list[str]:
|
|
273
329
|
"""Every advisory ID and alias that points at this vulnerability."""
|
|
274
330
|
rows = self._connection.execute(
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Failure causes for extraction errors (proposal §10, "failure analysis").
|
|
2
|
+
|
|
3
|
+
Every missed true symbol (a false negative) gets exactly one cause, checked in this order:
|
|
4
|
+
|
|
5
|
+
* ``not_changed``: the fix did not change the symbol, so no patch-based method can find it
|
|
6
|
+
(the ceiling ChangedFuncs also hits: "vulnerable function unchanged, callee changed");
|
|
7
|
+
* ``name_format``: the fix changed a definition that matches the symbol only on a dotted
|
|
8
|
+
boundary (GoVulnDB's ``decryptTicket`` for the method ``Conn.decryptTicket``): a naming
|
|
9
|
+
difference in the ground truth, not a missed function;
|
|
10
|
+
* ``dead_letter``: the symbol was changed, but the extraction failed;
|
|
11
|
+
* ``truncated``: changed, but dropped from the prompt to fit the token budget;
|
|
12
|
+
* ``model_missed``: shown to the model and judged not vulnerable.
|
|
13
|
+
|
|
14
|
+
Every false positive is either ``name_format`` (it matches a true symbol on a dotted
|
|
15
|
+
boundary) or ``model_false_positive``.
|
|
16
|
+
|
|
17
|
+
A report-level miss (no true symbol found: the safety metric) is attributed to its most
|
|
18
|
+
actionable cause: a shown-but-missed symbol first, then truncation, then the rest.
|
|
19
|
+
|
|
20
|
+
Not separated: "the fix adds a new helper". The replay data does not record which side of
|
|
21
|
+
the diff a candidate comes from, so such cases fall under ``model_false_positive``.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from collections import Counter
|
|
25
|
+
from collections.abc import Mapping, Set
|
|
26
|
+
|
|
27
|
+
from pydantic import BaseModel, ConfigDict
|
|
28
|
+
|
|
29
|
+
from patchsight.eval.safety import lenient_match
|
|
30
|
+
|
|
31
|
+
MISS_CAUSES = ("not_changed", "name_format", "dead_letter", "truncated", "model_missed")
|
|
32
|
+
REPORT_MISS_ORDER = ("model_missed", "truncated", "dead_letter", "name_format", "not_changed")
|
|
33
|
+
VALID = frozenset({"ok", "repaired"})
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ReportInput(BaseModel):
|
|
37
|
+
model_config = ConfigDict(frozen=True)
|
|
38
|
+
|
|
39
|
+
truth: frozenset[str]
|
|
40
|
+
changed: frozenset[str] # every changed function (ChangedFuncs)
|
|
41
|
+
shown: frozenset[str] # the candidates the prompt kept
|
|
42
|
+
status: str
|
|
43
|
+
predicted: frozenset[str]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class FailureBreakdown(BaseModel):
|
|
47
|
+
model_config = ConfigDict(frozen=True)
|
|
48
|
+
|
|
49
|
+
reports: int
|
|
50
|
+
truth_symbols: int
|
|
51
|
+
false_negatives: dict[str, int]
|
|
52
|
+
false_positives: dict[str, int]
|
|
53
|
+
report_misses: dict[str, int]
|
|
54
|
+
examples: dict[str, tuple[str, ...]] # cause -> report IDs (sorted)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def miss_cause(symbol: str, report: ReportInput) -> str:
|
|
58
|
+
if symbol not in report.changed:
|
|
59
|
+
if any(lenient_match(c, symbol) for c in report.changed):
|
|
60
|
+
return "name_format"
|
|
61
|
+
return "not_changed"
|
|
62
|
+
if report.status not in VALID:
|
|
63
|
+
return "dead_letter"
|
|
64
|
+
if symbol not in report.shown:
|
|
65
|
+
return "truncated"
|
|
66
|
+
return "model_missed"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def breakdown(reports: Mapping[str, ReportInput]) -> FailureBreakdown:
|
|
70
|
+
negatives: Counter[str] = Counter()
|
|
71
|
+
positives: Counter[str] = Counter()
|
|
72
|
+
misses: Counter[str] = Counter()
|
|
73
|
+
examples: dict[str, set[str]] = {}
|
|
74
|
+
total = 0
|
|
75
|
+
for rid, report in sorted(reports.items()):
|
|
76
|
+
total += len(report.truth)
|
|
77
|
+
causes = {miss_cause(s, report) for s in report.truth - report.predicted}
|
|
78
|
+
for symbol in report.truth - report.predicted:
|
|
79
|
+
cause = miss_cause(symbol, report)
|
|
80
|
+
negatives[cause] += 1
|
|
81
|
+
examples.setdefault(cause, set()).add(rid)
|
|
82
|
+
for symbol in report.predicted - report.truth:
|
|
83
|
+
cause = (
|
|
84
|
+
"name_format"
|
|
85
|
+
if any(lenient_match(symbol, t) for t in report.truth)
|
|
86
|
+
else "model_false_positive"
|
|
87
|
+
)
|
|
88
|
+
positives[cause] += 1
|
|
89
|
+
examples.setdefault(f"fp:{cause}", set()).add(rid)
|
|
90
|
+
if report.truth and not report.truth & report.predicted:
|
|
91
|
+
misses[next(c for c in REPORT_MISS_ORDER if c in causes)] += 1
|
|
92
|
+
return FailureBreakdown(
|
|
93
|
+
reports=len(reports),
|
|
94
|
+
truth_symbols=total,
|
|
95
|
+
false_negatives={c: negatives[c] for c in MISS_CAUSES},
|
|
96
|
+
false_positives={c: positives[c] for c in ("name_format", "model_false_positive")},
|
|
97
|
+
report_misses={c: misses[c] for c in REPORT_MISS_ORDER},
|
|
98
|
+
examples={k: tuple(sorted(v)) for k, v in sorted(examples.items())},
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def as_inputs(
|
|
103
|
+
truth: Mapping[str, Set[str]],
|
|
104
|
+
changed: Mapping[str, Set[str]],
|
|
105
|
+
shown: Mapping[str, Set[str]],
|
|
106
|
+
status: Mapping[str, str],
|
|
107
|
+
predicted: Mapping[str, Set[str]],
|
|
108
|
+
) -> dict[str, ReportInput]:
|
|
109
|
+
return {
|
|
110
|
+
r: ReportInput(
|
|
111
|
+
truth=frozenset(truth[r]),
|
|
112
|
+
changed=frozenset(changed.get(r, set())),
|
|
113
|
+
shown=frozenset(shown.get(r, set())),
|
|
114
|
+
status=status.get(r, "dead_letter"),
|
|
115
|
+
predicted=frozenset(predicted.get(r, set())),
|
|
116
|
+
)
|
|
117
|
+
for r in truth
|
|
118
|
+
}
|