patchsight 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchsight-0.3.0 → patchsight-0.4.0}/PKG-INFO +12 -3
- {patchsight-0.3.0 → patchsight-0.4.0}/docs/pypi.md +11 -2
- {patchsight-0.3.0 → patchsight-0.4.0}/pyproject.toml +1 -1
- {patchsight-0.3.0 → patchsight-0.4.0}/pyproject.toml.orig +1 -1
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/cli.py +13 -1
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/db.py +4 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/__main__.py +123 -4
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/extraction.py +1 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/extractor.py +2 -1
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/validate.py +24 -4
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/gate.py +20 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/config.py +9 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm_records.py +76 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/pypi_extract.py +1 -1
- patchsight-0.4.0/src/patchsight/scan/sarif.py +177 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/LICENSE +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/eval/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/eval/confidence.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/eval/failures.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/eval/metrics.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/eval/parity.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/eval/safety.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/names.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/bundle.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/changedfuncs.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/dunlap.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/replay.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/benchmarks/splits.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/build_db.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/commits.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/github.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/hunks.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/mapping.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/diffs/paths.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/prompt.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/extract/schema.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/fix_sample.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/http.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/aliases.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/ghsa_repo.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/osv.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/references.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/ingest/snapshot.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/kaggle_inputs.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/client.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/llm/tokens.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/pypi.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/update_db.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/pipeline/week1.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/py.typed +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/ast_scanner.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/dbcache.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/explain.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/__init__.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/poetry.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/requirements.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/toml_lock.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/lockfiles/uv.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/osv_client.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/report.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/run.py +0 -0
- {patchsight-0.3.0 → patchsight-0.4.0}/src/patchsight/scan/verdicts.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchsight
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Know which vulnerable Python dependencies your code actually calls: symbol-level triage, no LLM at scan time.
|
|
5
5
|
Keywords: security,vulnerability,dependencies,supply-chain,sca,static-analysis,reachability,osv,pypi,cli
|
|
6
6
|
Author: Niroj33
|
|
@@ -62,17 +62,26 @@ patchsight scan . # scan the lockfile and the code under .
|
|
|
62
62
|
|:---|:---|
|
|
63
63
|
| `--lockfile PATH` | Scan a specific lockfile: `requirements.txt`, `uv.lock` or `poetry.lock` (default: the first found, in that order) |
|
|
64
64
|
| `--format json` | Machine-readable output |
|
|
65
|
+
| `--format sarif` | SARIF 2.1.0 for GitHub code scanning and other SARIF readers |
|
|
65
66
|
| `--fail-on called` | Exit 1 if any alert is CALLED: a CI gate (`unknown` also fails on UNKNOWN) |
|
|
66
67
|
| `--db PATH` | Use a specific database file (or set `PATCHSIGHT_DB`) |
|
|
67
68
|
| `patchsight explain ID` | Why the database lists an advisory's symbols: confidence, models, rationale and the changed lines |
|
|
68
69
|
| `patchsight db info` | Show which database a scan uses, and its age |
|
|
69
70
|
|
|
71
|
+
In GitHub Actions, the [PatchSight action](https://github.com/professor3333/PatchSight#in-ci-the-github-action) scans and uploads the SARIF report to code scanning:
|
|
72
|
+
|
|
73
|
+
```yaml
|
|
74
|
+
- uses: professor3333/PatchSight@v0.4.0
|
|
75
|
+
with:
|
|
76
|
+
fail-on: called
|
|
77
|
+
```
|
|
78
|
+
|
|
70
79
|
## Verdicts
|
|
71
80
|
|
|
72
81
|
| | Verdict | Meaning |
|
|
73
82
|
|:---:|:---|:---|
|
|
74
83
|
| 🔴 | **CALLED** | A reference in your code resolves to a vulnerable function, its class, or an entry point that reaches it. **Fix these first.** |
|
|
75
|
-
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data. |
|
|
84
|
+
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data, which is [withheld for now](https://github.com/professor3333/PatchSight/blob/main/docs/adr/0008-withhold-high-confidence.md). |
|
|
76
85
|
| ⚪ | **NOT IMPORTED** | Your code never imports the package directly. It may still be used through other dependencies. |
|
|
77
86
|
| 🟡 | **UNKNOWN** | No symbol data, only low-confidence data, or dynamic code the scanner can't follow. |
|
|
78
87
|
|
|
@@ -93,7 +102,7 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
93
102
|
## Status: early release
|
|
94
103
|
|
|
95
104
|
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
96
|
-
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached.
|
|
105
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. High-confidence records are withheld until the confidence rule meets its safety target, so expect UNKNOWN where a later release may say IMPORTED, NOT CALLED.
|
|
97
106
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
98
107
|
|
|
99
108
|
## Links
|
|
@@ -24,17 +24,26 @@ patchsight scan . # scan the lockfile and the code under .
|
|
|
24
24
|
|:---|:---|
|
|
25
25
|
| `--lockfile PATH` | Scan a specific lockfile: `requirements.txt`, `uv.lock` or `poetry.lock` (default: the first found, in that order) |
|
|
26
26
|
| `--format json` | Machine-readable output |
|
|
27
|
+
| `--format sarif` | SARIF 2.1.0 for GitHub code scanning and other SARIF readers |
|
|
27
28
|
| `--fail-on called` | Exit 1 if any alert is CALLED: a CI gate (`unknown` also fails on UNKNOWN) |
|
|
28
29
|
| `--db PATH` | Use a specific database file (or set `PATCHSIGHT_DB`) |
|
|
29
30
|
| `patchsight explain ID` | Why the database lists an advisory's symbols: confidence, models, rationale and the changed lines |
|
|
30
31
|
| `patchsight db info` | Show which database a scan uses, and its age |
|
|
31
32
|
|
|
33
|
+
In GitHub Actions, the [PatchSight action](https://github.com/professor3333/PatchSight#in-ci-the-github-action) scans and uploads the SARIF report to code scanning:
|
|
34
|
+
|
|
35
|
+
```yaml
|
|
36
|
+
- uses: professor3333/PatchSight@v0.4.0
|
|
37
|
+
with:
|
|
38
|
+
fail-on: called
|
|
39
|
+
```
|
|
40
|
+
|
|
32
41
|
## Verdicts
|
|
33
42
|
|
|
34
43
|
| | Verdict | Meaning |
|
|
35
44
|
|:---:|:---|:---|
|
|
36
45
|
| 🔴 | **CALLED** | A reference in your code resolves to a vulnerable function, its class, or an entry point that reaches it. **Fix these first.** |
|
|
37
|
-
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data. |
|
|
46
|
+
| 🟢 | **IMPORTED, NOT CALLED** | The package is imported, but no vulnerable symbol is referenced. Given only for high-confidence data, which is [withheld for now](https://github.com/professor3333/PatchSight/blob/main/docs/adr/0008-withhold-high-confidence.md). |
|
|
38
47
|
| ⚪ | **NOT IMPORTED** | Your code never imports the package directly. It may still be used through other dependencies. |
|
|
39
48
|
| 🟡 | **UNKNOWN** | No symbol data, only low-confidence data, or dynamic code the scanner can't follow. |
|
|
40
49
|
|
|
@@ -55,7 +64,7 @@ The F1 95% interval is 0.62 to 0.74. The reports and a one-command offline repro
|
|
|
55
64
|
## Status: early release
|
|
56
65
|
|
|
57
66
|
- Reads `requirements.txt` (exact `==` pins only), `uv.lock` and `poetry.lock`. Packages without a PyPI release version (unpinned requirements, git, URL or local sources) are listed, not guessed.
|
|
58
|
-
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached.
|
|
67
|
+
- The [symbol database](https://github.com/professor3333/PatchSight/releases) is updated every night and covers about 3,500 PyPI vulnerabilities, all at **low confidence** so far: every function the fix changed, plus the model's picks where the backfill has reached. High-confidence records are withheld until the confidence rule meets its safety target, so expect UNKNOWN where a later release may say IMPORTED, NOT CALLED.
|
|
59
68
|
- Transitive use and dynamic code (`getattr` with a computed name, `importlib`, star imports) aren't analysed. They give UNKNOWN or NOT IMPORTED, never a false "not called".
|
|
60
69
|
|
|
61
70
|
## Links
|
|
@@ -102,7 +102,9 @@ def scan(
|
|
|
102
102
|
Path | None,
|
|
103
103
|
typer.Option("--db", help="Symbol database (default: the one 'db update' downloaded)."),
|
|
104
104
|
] = None,
|
|
105
|
-
output: Annotated[
|
|
105
|
+
output: Annotated[
|
|
106
|
+
str, typer.Option("--format", help="text, json, or sarif (for code scanning).")
|
|
107
|
+
] = "text",
|
|
106
108
|
fail_on: Annotated[
|
|
107
109
|
str, typer.Option(help="Exit 1 when an alert is: called, unknown (or called), or never.")
|
|
108
110
|
] = "never",
|
|
@@ -131,6 +133,16 @@ def scan(
|
|
|
131
133
|
shown = lockfile.as_posix()
|
|
132
134
|
if output == "json":
|
|
133
135
|
typer.echo(render_json(alerts, dependencies, shown, meta))
|
|
136
|
+
elif output == "sarif":
|
|
137
|
+
import os
|
|
138
|
+
|
|
139
|
+
from patchsight.scan.sarif import render_sarif
|
|
140
|
+
|
|
141
|
+
# Code locations are relative to the scanned directory; SARIF wants them relative to
|
|
142
|
+
# where the scan runs (the repository root, in CI).
|
|
143
|
+
root = path if path.is_dir() else path.parent
|
|
144
|
+
base = os.path.relpath(root.resolve(), Path.cwd().resolve())
|
|
145
|
+
typer.echo(render_sarif(alerts, dependencies, shown, meta, base))
|
|
134
146
|
else:
|
|
135
147
|
typer.echo(render_text(alerts, dependencies, shown, meta))
|
|
136
148
|
failing = {"called": {"CALLED"}, "unknown": {"CALLED", "UNKNOWN"}}.get(fail_on, set())
|
|
@@ -295,6 +295,10 @@ class SymbolDatabase:
|
|
|
295
295
|
).fetchone()
|
|
296
296
|
return str(row[0]) if row else None
|
|
297
297
|
|
|
298
|
+
def all_records(self) -> list[Record]:
|
|
299
|
+
"""Every record, ordered by vulnerability and package."""
|
|
300
|
+
return [r for v in self.vulnerabilities() for r in self.records_for(v)]
|
|
301
|
+
|
|
298
302
|
def confidence_counts(self) -> dict[str, int]:
|
|
299
303
|
"""Records by confidence tier."""
|
|
300
304
|
rows = self._connection.execute(
|
|
@@ -619,6 +619,13 @@ def confidence_report_command(
|
|
|
619
619
|
primary: Annotated[str, typer.Option(help="Extractor run tag (replay bundle).")],
|
|
620
620
|
secondary: Annotated[str, typer.Option(help="Second-opinion run tag (replay bundle).")],
|
|
621
621
|
part: Annotated[str, typer.Option("--split", help="Split part: dev, test or all.")] = "dev",
|
|
622
|
+
variant: Annotated[
|
|
623
|
+
str | None,
|
|
624
|
+
typer.Option(
|
|
625
|
+
help="Experiment variant: read entry points from the reports ending in -VARIANT, "
|
|
626
|
+
"and write the report under that suffix."
|
|
627
|
+
),
|
|
628
|
+
] = None,
|
|
622
629
|
root: Root = Path(),
|
|
623
630
|
) -> None:
|
|
624
631
|
"""Compare confidence rules, in the benchmark and safety views, from committed files."""
|
|
@@ -635,8 +642,9 @@ def confidence_report_command(
|
|
|
635
642
|
for tag in (primary, secondary):
|
|
636
643
|
run = replay.load(root / DUNLAP_BUNDLE.parent / f"extraction-{tag}.json")
|
|
637
644
|
# Validated entry points come from the run's committed report (any part that has them).
|
|
645
|
+
suffix = f"{tag}-{variant}" if variant else tag
|
|
638
646
|
entry: dict[str, frozenset[str]] = {}
|
|
639
|
-
for source in sorted(reports_dir.glob(f"extraction-dunlap-*-{
|
|
647
|
+
for source in sorted(reports_dir.glob(f"extraction-dunlap-*-{suffix}.json")):
|
|
640
648
|
for outcome in extraction.ExtractionRun.model_validate_json(
|
|
641
649
|
source.read_text("utf-8")
|
|
642
650
|
).outcomes:
|
|
@@ -678,6 +686,11 @@ def confidence_report_command(
|
|
|
678
686
|
f"- Extractor: `{runs[primary][0]}` (`{primary}`)",
|
|
679
687
|
f"- Second opinion: `{runs[secondary][0]}` (`{secondary}`)",
|
|
680
688
|
f"- Split `{frozen.split.name}:{part}` (SHA-256 `{frozen.sha256}`), {len(first)} reports",
|
|
689
|
+
*(
|
|
690
|
+
[f"- Variant `{variant}`: entry points from the `-{variant}` extraction reports"]
|
|
691
|
+
if variant
|
|
692
|
+
else []
|
|
693
|
+
),
|
|
681
694
|
"",
|
|
682
695
|
"Two views. The **benchmark view** compares function names exactly and ignores entry "
|
|
683
696
|
"points, so it stays comparable with Dunlap et al. The **safety view** scores what the "
|
|
@@ -727,12 +740,13 @@ def confidence_report_command(
|
|
|
727
740
|
reasons = ", ".join(f"{k} {v}" for k, v in r.low_reasons.items())
|
|
728
741
|
lines.append(f"- `{r.rule}`{', public' if r.require_public else ''}: {reasons}")
|
|
729
742
|
lines.append("")
|
|
730
|
-
|
|
743
|
+
name = f"confidence-dunlap-{part}-{primary}-vs-{secondary}" + (f"-{variant}" if variant else "")
|
|
744
|
+
target = reports_dir / name
|
|
731
745
|
target.with_name(target.name + ".md").write_text("\n".join(lines), "utf-8")
|
|
732
746
|
target.with_name(target.name + ".json").write_text(
|
|
733
747
|
json.dumps([r.model_dump(mode="json") for r in results], indent=2) + "\n", "utf-8"
|
|
734
748
|
)
|
|
735
|
-
typer.echo("\n".join(lines[10:]))
|
|
749
|
+
typer.echo("\n".join(lines[10 + bool(variant) :]))
|
|
736
750
|
|
|
737
751
|
|
|
738
752
|
@app.command("parity-report")
|
|
@@ -1036,6 +1050,69 @@ def update_db_command(
|
|
|
1036
1050
|
typer.echo(f"LLM merge: {merged}")
|
|
1037
1051
|
|
|
1038
1052
|
|
|
1053
|
+
@app.command("merge-llm")
|
|
1054
|
+
def merge_llm_command(
|
|
1055
|
+
database: Annotated[Path, typer.Argument(help="The database to merge into.")],
|
|
1056
|
+
version: Annotated[str, typer.Option(help="Version of the result, e.g. 2026.11.17.1.")],
|
|
1057
|
+
llm: Annotated[
|
|
1058
|
+
list[Path], typer.Option(help="Extractor outcomes (pypi-extract JSONL); repeatable.")
|
|
1059
|
+
],
|
|
1060
|
+
second: Annotated[
|
|
1061
|
+
list[Path] | None, typer.Option(help="Second-opinion outcomes; repeatable.")
|
|
1062
|
+
] = None,
|
|
1063
|
+
root: Root = Path(),
|
|
1064
|
+
out: Annotated[Path, typer.Option(help="Output path, without extension.")] = Path(
|
|
1065
|
+
"data/build/patchsight-merged"
|
|
1066
|
+
),
|
|
1067
|
+
) -> None:
|
|
1068
|
+
"""Merge new LLM extractions into an existing database (the backfill, shard by shard)."""
|
|
1069
|
+
from patchsight.pipeline import llm_records, pypi_extract
|
|
1070
|
+
|
|
1071
|
+
source = db.SymbolDatabase(root / database)
|
|
1072
|
+
try:
|
|
1073
|
+
records, aliases = source.all_records(), source.alias_map()
|
|
1074
|
+
builds, meta = list(source.build_ledger().values()), dict(source.meta)
|
|
1075
|
+
finally:
|
|
1076
|
+
source.close()
|
|
1077
|
+
if not builds:
|
|
1078
|
+
raise typer.BadParameter("the database has no builds ledger; build it with build-db")
|
|
1079
|
+
primaries: dict[str, pypi_extract.PyPIExtraction] = {}
|
|
1080
|
+
for path in llm:
|
|
1081
|
+
primaries.update(llm_records.load(root / path))
|
|
1082
|
+
secondaries: dict[str, pypi_extract.PyPIExtraction] = {}
|
|
1083
|
+
for path in second or []:
|
|
1084
|
+
secondaries.update(llm_records.load(root / path))
|
|
1085
|
+
merged, outcomes = llm_records.apply(records, primaries, secondaries)
|
|
1086
|
+
report = llm_records.merge_report(
|
|
1087
|
+
records, merged, primaries, secondaries, outcomes, meta.get("db_version", "?"), version
|
|
1088
|
+
)
|
|
1089
|
+
commit = subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
1090
|
+
["git", "-C", str(root), "rev-parse", "HEAD"], # noqa: S607
|
|
1091
|
+
capture_output=True, text=True, check=False,
|
|
1092
|
+
).stdout.strip() # fmt: skip
|
|
1093
|
+
meta.pop("updated_from", None) # the source's own origin; this build is a merge
|
|
1094
|
+
meta.update(
|
|
1095
|
+
{
|
|
1096
|
+
"db_version": version,
|
|
1097
|
+
"built_at": datetime.now(UTC).isoformat(),
|
|
1098
|
+
"builder": "llm-v1",
|
|
1099
|
+
"code_commit": commit,
|
|
1100
|
+
"merged_into": meta.get("db_version", "?"),
|
|
1101
|
+
"records_by_tier": json.dumps(dict(Counter(r.confidence for r in merged))),
|
|
1102
|
+
}
|
|
1103
|
+
)
|
|
1104
|
+
meta.pop("schema_version", None) # db.write sets it
|
|
1105
|
+
target = root / out
|
|
1106
|
+
db.write(target.with_name(target.name + ".sqlite"), merged, aliases, meta, builds=builds)
|
|
1107
|
+
target.with_name(target.name + "-merge.json").write_text(
|
|
1108
|
+
report.model_dump_json(indent=2) + "\n", "utf-8"
|
|
1109
|
+
)
|
|
1110
|
+
typer.echo(
|
|
1111
|
+
f"db {version} merged into {meta['merged_into']}: {len(merged)} records; "
|
|
1112
|
+
f"{dict(Counter(r.confidence for r in merged))}; {outcomes}"
|
|
1113
|
+
)
|
|
1114
|
+
|
|
1115
|
+
|
|
1039
1116
|
@app.command("gate-db")
|
|
1040
1117
|
def gate_db_command(
|
|
1041
1118
|
candidate: Annotated[Path, typer.Argument(help="The candidate database.")],
|
|
@@ -1077,6 +1154,11 @@ CC BY 4.0.
|
|
|
1077
1154
|
lines to function names with tree-sitter, and, where shown, had language models choose the \
|
|
1078
1155
|
vulnerable ones. It stores those names with file paths, line ranges and commit SHAs. No \
|
|
1079
1156
|
third-party source code is included.
|
|
1157
|
+
|
|
1158
|
+
**Licence:** the PatchSight symbol database is licensed under \
|
|
1159
|
+
[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). If you share it, or data derived from \
|
|
1160
|
+
it, credit "PatchSight symbol database" with the release, keep the attribution above, and say \
|
|
1161
|
+
what you changed.
|
|
1080
1162
|
"""
|
|
1081
1163
|
|
|
1082
1164
|
|
|
@@ -1087,6 +1169,7 @@ def release_notes_command(
|
|
|
1087
1169
|
"releases/latest.json"
|
|
1088
1170
|
),
|
|
1089
1171
|
update: Annotated[Path | None, typer.Option(help="The update-db report, if any.")] = None,
|
|
1172
|
+
merge: Annotated[Path | None, typer.Option(help="The merge-llm report, if any.")] = None,
|
|
1090
1173
|
gate: Annotated[Path | None, typer.Option(help="The gate-db report, if any.")] = None,
|
|
1091
1174
|
root: Root = Path(),
|
|
1092
1175
|
out: Annotated[Path, typer.Option(help="Notes file to write.")] = Path(
|
|
@@ -1095,6 +1178,7 @@ def release_notes_command(
|
|
|
1095
1178
|
) -> None:
|
|
1096
1179
|
"""Write release notes for a database release, with provenance and attribution."""
|
|
1097
1180
|
from patchsight.pipeline.gate import GateReport
|
|
1181
|
+
from patchsight.pipeline.llm_records import MergeReport
|
|
1098
1182
|
from patchsight.pipeline.update_db import UpdateReport
|
|
1099
1183
|
|
|
1100
1184
|
opened = db.SymbolDatabase(root / database)
|
|
@@ -1105,7 +1189,13 @@ def release_notes_command(
|
|
|
1105
1189
|
opened.close()
|
|
1106
1190
|
published = dbcache.Pointer.model_validate_json((root / pointer).read_text("utf-8"))
|
|
1107
1191
|
version = meta.get("db_version", "?")
|
|
1108
|
-
origin =
|
|
1192
|
+
origin = (
|
|
1193
|
+
f", updated from {meta['updated_from']}"
|
|
1194
|
+
if "updated_from" in meta
|
|
1195
|
+
else f", {meta['merged_into']} with new model picks"
|
|
1196
|
+
if "merged_into" in meta
|
|
1197
|
+
else ""
|
|
1198
|
+
)
|
|
1109
1199
|
lines = [
|
|
1110
1200
|
f"Symbol database **{version}**{origin}.",
|
|
1111
1201
|
"",
|
|
@@ -1113,6 +1203,35 @@ def release_notes_command(
|
|
|
1113
1203
|
"patchsight db update # downloads this release and verifies its SHA-256",
|
|
1114
1204
|
"```",
|
|
1115
1205
|
"",
|
|
1206
|
+
]
|
|
1207
|
+
if merge is not None:
|
|
1208
|
+
merged = MergeReport.model_validate_json((root / merge).read_text("utf-8"))
|
|
1209
|
+
second = (
|
|
1210
|
+
"; second opinion " + ", ".join(f"`{m}`" for m in merged.second_opinion_models)
|
|
1211
|
+
if merged.second_opinion_models
|
|
1212
|
+
else ""
|
|
1213
|
+
)
|
|
1214
|
+
lines += [
|
|
1215
|
+
f"## Changes since {merged.previous_version}",
|
|
1216
|
+
"",
|
|
1217
|
+
"- Merges model picks from "
|
|
1218
|
+
+ ", ".join(f"`{m}`" for m in merged.models)
|
|
1219
|
+
+ " (prompt "
|
|
1220
|
+
+ ", ".join(f"`{p}`" for p in merged.prompt_versions)
|
|
1221
|
+
+ f"){second}.",
|
|
1222
|
+
f"- {merged.after.vulnerabilities:,} vulnerabilities ({merged.after.records:,} "
|
|
1223
|
+
"records) now carry model picks with a rationale and cited hunks, up from "
|
|
1224
|
+
f"{merged.before.vulnerabilities:,} ({merged.before.records:,} records).",
|
|
1225
|
+
"- No record is high confidence yet, so matching still uses every changed "
|
|
1226
|
+
"definition. `patchsight explain <id>` shows the picks."
|
|
1227
|
+
if not merged.tiers.get("high")
|
|
1228
|
+
else f"- {merged.tiers['high']:,} "
|
|
1229
|
+
+ ("record is" if merged.tiers["high"] == 1 else "records are")
|
|
1230
|
+
+ " high confidence. `patchsight explain <id>` shows the picks and why each tier "
|
|
1231
|
+
"was chosen.",
|
|
1232
|
+
"",
|
|
1233
|
+
]
|
|
1234
|
+
lines += [
|
|
1116
1235
|
"## Contents",
|
|
1117
1236
|
"",
|
|
1118
1237
|
f"- **{vulnerabilities:,} PyPI vulnerabilities**, {total:,} records: "
|
|
@@ -118,6 +118,7 @@ def extract(
|
|
|
118
118
|
counter: TokenCounter,
|
|
119
119
|
max_hunk_lines: int,
|
|
120
120
|
max_candidates: int | None = None,
|
|
121
|
+
dotted_entry_points: bool = False,
|
|
121
122
|
) -> ExtractionOutcome:
|
|
122
123
|
prompt = render(
|
|
123
124
|
template, advisory, inputs, budget_tokens, counter, max_hunk_lines, max_candidates
|
|
@@ -156,7 +157,7 @@ def extract(
|
|
|
156
157
|
continue
|
|
157
158
|
responses.append(response)
|
|
158
159
|
try:
|
|
159
|
-
result = validate(response.content, prompt, known_names)
|
|
160
|
+
result = validate(response.content, prompt, known_names, dotted_entry_points)
|
|
160
161
|
break
|
|
161
162
|
except ValidationFailedError as failure:
|
|
162
163
|
if attempt == 0:
|
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
* Every candidate gets exactly one judgement. Unknown IDs are counted as invented.
|
|
5
5
|
* Every vulnerable judgement cites at least one hunk, and every cited hunk must be shown
|
|
6
6
|
under that candidate.
|
|
7
|
-
* Entry points survive only if they name a definition known from the fetched source
|
|
7
|
+
* Entry points survive only if they name a definition known from the fetched source:
|
|
8
|
+
exactly, or, when ``dotted`` is on, by matching exactly one definition on a dotted boundary
|
|
9
|
+
(``SetValue`` names ``Paved.SetValue``; ``yaml.load`` names ``load``). An ambiguous name is
|
|
10
|
+
dropped. A resolved entry point is kept as the definition's own name.
|
|
8
11
|
"""
|
|
9
12
|
|
|
10
13
|
import json
|
|
@@ -13,6 +16,7 @@ from collections.abc import Set
|
|
|
13
16
|
|
|
14
17
|
from pydantic import BaseModel, ConfigDict, ValidationError
|
|
15
18
|
|
|
19
|
+
from patchsight.eval.safety import lenient_match
|
|
16
20
|
from patchsight.pipeline.extract.prompt import RenderedPrompt
|
|
17
21
|
from patchsight.pipeline.extract.schema import Extraction
|
|
18
22
|
|
|
@@ -34,7 +38,19 @@ class ValidatedExtraction(BaseModel):
|
|
|
34
38
|
entry_points_dropped: tuple[str, ...]
|
|
35
39
|
|
|
36
40
|
|
|
37
|
-
def
|
|
41
|
+
def resolve_entry_point(name: str, allowed: Set[str], dotted: bool) -> str | None:
|
|
42
|
+
"""The definition an entry point names, or None when it names none (or several)."""
|
|
43
|
+
if name in allowed:
|
|
44
|
+
return name
|
|
45
|
+
if not dotted:
|
|
46
|
+
return None
|
|
47
|
+
matches = [d for d in allowed if lenient_match(name, d)]
|
|
48
|
+
return matches[0] if len(matches) == 1 else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def validate(
|
|
52
|
+
content: str, prompt: RenderedPrompt, known_names: Set[str], dotted_entry_points: bool = False
|
|
53
|
+
) -> ValidatedExtraction:
|
|
38
54
|
try:
|
|
39
55
|
extraction = Extraction.model_validate(json.loads(content))
|
|
40
56
|
except (json.JSONDecodeError, ValidationError) as error:
|
|
@@ -69,8 +85,12 @@ def validate(content: str, prompt: RenderedPrompt, known_names: Set[str]) -> Val
|
|
|
69
85
|
raise ValidationFailedError(errors, len(invented))
|
|
70
86
|
|
|
71
87
|
allowed = set(known_names) | {c.name for c in prompt.candidates}
|
|
72
|
-
|
|
73
|
-
|
|
88
|
+
resolved = {
|
|
89
|
+
e: resolve_entry_point(e, allowed, dotted_entry_points)
|
|
90
|
+
for e in extraction.public_entry_points
|
|
91
|
+
}
|
|
92
|
+
kept = tuple(sorted({d for d in resolved.values() if d is not None}))
|
|
93
|
+
dropped = tuple(sorted({e for e, d in resolved.items() if d is None}))
|
|
74
94
|
judged = {item.candidate_id: item for item in extraction.items}
|
|
75
95
|
return ValidatedExtraction(
|
|
76
96
|
vulnerable_ids=tuple(c.id for c in prompt.candidates if judged[c.id].is_vulnerable),
|
|
@@ -13,12 +13,15 @@ workflow then opens an issue with this report instead.
|
|
|
13
13
|
``MAX_LOST_SHARE`` of the previous ones, or ``MIN_LOST_ALLOWED``, whichever is larger).
|
|
14
14
|
A bigger drop means a broken snapshot or builder, not real withdrawals.
|
|
15
15
|
* ``high_kept``: a high-confidence record whose extraction inputs did not change stays high.
|
|
16
|
+
* ``high_withheld``: no record is high confidence while ADR 0008 withholds the tier, however
|
|
17
|
+
the database was built.
|
|
16
18
|
"""
|
|
17
19
|
|
|
18
20
|
from pydantic import BaseModel, ConfigDict
|
|
19
21
|
|
|
20
22
|
from patchsight.db import SymbolDatabase
|
|
21
23
|
from patchsight.pipeline.build_db import MAX_SYMBOLS
|
|
24
|
+
from patchsight.pipeline.llm_records import PUBLISH_HIGH
|
|
22
25
|
|
|
23
26
|
MAX_LOST_SHARE = 0.02
|
|
24
27
|
MIN_LOST_ALLOWED = 10
|
|
@@ -126,6 +129,23 @@ def run(candidate: SymbolDatabase, previous: SymbolDatabase) -> GateReport:
|
|
|
126
129
|
+ (f"; demoted: {_sample(demoted)}" if demoted else ""),
|
|
127
130
|
)
|
|
128
131
|
)
|
|
132
|
+
|
|
133
|
+
high = [f"{r.vulnerability}/{r.package}" for rs in records.values() for r in rs
|
|
134
|
+
if r.confidence == "high"] # fmt: skip
|
|
135
|
+
checks.append(
|
|
136
|
+
Check(
|
|
137
|
+
name="high_withheld",
|
|
138
|
+
passed=PUBLISH_HIGH or not high,
|
|
139
|
+
detail=f"{len(high)} high-confidence records"
|
|
140
|
+
+ (
|
|
141
|
+
""
|
|
142
|
+
if PUBLISH_HIGH
|
|
143
|
+
else " (ADR 0008 withholds the tier)"
|
|
144
|
+
if not high
|
|
145
|
+
else f" while ADR 0008 withholds the tier: {_sample(high)}"
|
|
146
|
+
),
|
|
147
|
+
)
|
|
148
|
+
)
|
|
129
149
|
return GateReport(
|
|
130
150
|
candidate_version=candidate.meta.get("db_version", "?"),
|
|
131
151
|
previous_version=previous.meta.get("db_version", "?"),
|
|
@@ -42,6 +42,14 @@ class BudgetConfig(BaseModel):
|
|
|
42
42
|
output_tokens_per_candidate: int | None = None
|
|
43
43
|
|
|
44
44
|
|
|
45
|
+
class ValidationConfig(BaseModel):
|
|
46
|
+
model_config = ConfigDict(frozen=True)
|
|
47
|
+
|
|
48
|
+
# Accept an entry point that matches no definition exactly but exactly one on a dotted
|
|
49
|
+
# boundary (``SetValue`` for ``Paved.SetValue``), kept as that definition's name.
|
|
50
|
+
dotted_entry_points: bool = False
|
|
51
|
+
|
|
52
|
+
|
|
45
53
|
class LLMConfig(BaseModel):
|
|
46
54
|
model_config = ConfigDict(frozen=True)
|
|
47
55
|
|
|
@@ -49,6 +57,7 @@ class LLMConfig(BaseModel):
|
|
|
49
57
|
models: dict[str, ModelConfig]
|
|
50
58
|
budget: BudgetConfig
|
|
51
59
|
tokenizer: TokenizerConfig
|
|
60
|
+
validation: ValidationConfig = ValidationConfig()
|
|
52
61
|
|
|
53
62
|
def prompt_budget(self, role: str) -> int:
|
|
54
63
|
"""Largest prompt, in exact tokens, that still fits the repair round under the caps.
|
|
@@ -11,16 +11,32 @@ extractor has a valid answer made from the same inputs:
|
|
|
11
11
|
|
|
12
12
|
An extractor answer that failed validation, or was made from different inputs (its
|
|
13
13
|
``input_hash`` no longer matches the advisory), leaves the fallback record unchanged.
|
|
14
|
+
|
|
15
|
+
**High confidence is withheld** (ADR 0008): the rule missed its safety target in its declared
|
|
16
|
+
test evaluation, so a record the rule would make high is kept low, with every changed
|
|
17
|
+
definition, and its tier reason says so. Lifting this takes a new ADR, not a flag.
|
|
18
|
+
|
|
19
|
+
A record that is already high confidence is never re-merged: it no longer stores every
|
|
20
|
+
changed definition, so a lower tier could not be rebuilt from it. ``merge`` can therefore run
|
|
21
|
+
again over a published database (``merge-llm``) as new extractions arrive.
|
|
14
22
|
"""
|
|
15
23
|
|
|
16
24
|
from collections import Counter
|
|
17
25
|
from collections.abc import Iterable, Mapping, Sequence
|
|
18
26
|
from pathlib import Path
|
|
19
27
|
|
|
28
|
+
from pydantic import BaseModel, ConfigDict
|
|
29
|
+
|
|
20
30
|
from patchsight.db import Record, Symbol, SymbolEvidence
|
|
21
31
|
from patchsight.eval.confidence import VALID, Answer, Decision, decide, is_public
|
|
22
32
|
from patchsight.pipeline.pypi_extract import PyPIExtraction
|
|
23
33
|
|
|
34
|
+
# ADR 0008: in the declared test evaluation (ADR 0004), 7.0% of high-confidence reports missed
|
|
35
|
+
# the vulnerable symbol, against a target of 2% or less. Until a revised rule meets the target
|
|
36
|
+
# on data not used to choose it, published databases hold no high-confidence records.
|
|
37
|
+
PUBLISH_HIGH = False
|
|
38
|
+
WITHHELD = "high confidence withheld (ADR 0008)"
|
|
39
|
+
|
|
24
40
|
|
|
25
41
|
def load(path: Path) -> dict[str, PyPIExtraction]:
|
|
26
42
|
"""A pypi-extract JSONL file, keyed by vulnerability."""
|
|
@@ -60,8 +76,11 @@ def merge(
|
|
|
60
76
|
fallback: Sequence[Record],
|
|
61
77
|
primary: PyPIExtraction | None,
|
|
62
78
|
secondary: PyPIExtraction | None,
|
|
79
|
+
publish_high: bool = PUBLISH_HIGH,
|
|
63
80
|
) -> tuple[list[Record], str]:
|
|
64
81
|
"""The records for one vulnerability, and what happened (for the build report)."""
|
|
82
|
+
if any(r.confidence == "high" for r in fallback):
|
|
83
|
+
return list(fallback), "kept: already high confidence"
|
|
65
84
|
if primary is None:
|
|
66
85
|
return list(fallback), "fallback: no extraction"
|
|
67
86
|
if any(r.input_hash != primary.input_hash for r in fallback):
|
|
@@ -73,6 +92,8 @@ def merge(
|
|
|
73
92
|
decision: Decision = decide(
|
|
74
93
|
answer(primary), answer(secondary) if secondary else None, language="python"
|
|
75
94
|
)
|
|
95
|
+
if decision.tier == "high" and not publish_high:
|
|
96
|
+
decision = Decision(tier="low", reason=f"{WITHHELD}: {decision.reason}", agreed=frozenset())
|
|
76
97
|
valid_second = secondary is not None and secondary.outcome.status in VALID
|
|
77
98
|
entry_points = set(primary.entry_points)
|
|
78
99
|
if valid_second and secondary is not None:
|
|
@@ -115,6 +136,7 @@ def apply(
|
|
|
115
136
|
records: Iterable[Record],
|
|
116
137
|
primaries: Mapping[str, PyPIExtraction],
|
|
117
138
|
secondaries: Mapping[str, PyPIExtraction],
|
|
139
|
+
publish_high: bool = PUBLISH_HIGH,
|
|
118
140
|
) -> tuple[list[Record], dict[str, int]]:
|
|
119
141
|
"""Merge extraction into fallback records. Returns the records and outcome counts."""
|
|
120
142
|
by_vulnerability: dict[str, list[Record]] = {}
|
|
@@ -127,7 +149,61 @@ def apply(
|
|
|
127
149
|
by_vulnerability[vulnerability],
|
|
128
150
|
primaries.get(vulnerability),
|
|
129
151
|
secondaries.get(vulnerability),
|
|
152
|
+
publish_high,
|
|
130
153
|
)
|
|
131
154
|
merged.extend(result)
|
|
132
155
|
outcomes[outcome] += 1
|
|
133
156
|
return merged, dict(sorted(outcomes.items()))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class Picks(BaseModel):
|
|
160
|
+
"""How much of a database carries model picks (records whose source is the LLM)."""
|
|
161
|
+
|
|
162
|
+
model_config = ConfigDict(frozen=True)
|
|
163
|
+
|
|
164
|
+
vulnerabilities: int
|
|
165
|
+
records: int
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def picks(records: Iterable[Record]) -> Picks:
|
|
169
|
+
chosen = [r for r in records if r.source == "llm"]
|
|
170
|
+
return Picks(vulnerabilities=len({r.vulnerability for r in chosen}), records=len(chosen))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class MergeReport(BaseModel):
|
|
174
|
+
"""What one ``merge-llm`` run changed, for the release notes."""
|
|
175
|
+
|
|
176
|
+
model_config = ConfigDict(frozen=True)
|
|
177
|
+
|
|
178
|
+
previous_version: str
|
|
179
|
+
db_version: str
|
|
180
|
+
models: tuple[str, ...] # extractor models whose answers were offered
|
|
181
|
+
second_opinion_models: tuple[str, ...]
|
|
182
|
+
prompt_versions: tuple[str, ...]
|
|
183
|
+
before: Picks
|
|
184
|
+
after: Picks
|
|
185
|
+
tiers: dict[str, int] # records by confidence, after the merge
|
|
186
|
+
outcomes: dict[str, int] # per vulnerability, as returned by apply
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def merge_report(
|
|
190
|
+
before: Sequence[Record],
|
|
191
|
+
after: Sequence[Record],
|
|
192
|
+
primaries: Mapping[str, PyPIExtraction],
|
|
193
|
+
secondaries: Mapping[str, PyPIExtraction],
|
|
194
|
+
outcomes: Mapping[str, int],
|
|
195
|
+
previous_version: str,
|
|
196
|
+
db_version: str,
|
|
197
|
+
) -> MergeReport:
|
|
198
|
+
offered = [*primaries.values(), *secondaries.values()]
|
|
199
|
+
return MergeReport(
|
|
200
|
+
previous_version=previous_version,
|
|
201
|
+
db_version=db_version,
|
|
202
|
+
models=tuple(sorted({e.outcome.model for e in primaries.values()})),
|
|
203
|
+
second_opinion_models=tuple(sorted({e.outcome.model for e in secondaries.values()})),
|
|
204
|
+
prompt_versions=tuple(sorted({e.outcome.prompt_version for e in offered})),
|
|
205
|
+
before=picks(before),
|
|
206
|
+
after=picks(after),
|
|
207
|
+
tiers=dict(sorted(Counter(r.confidence for r in after).items())),
|
|
208
|
+
outcomes=dict(sorted(outcomes.items())),
|
|
209
|
+
)
|
|
@@ -237,7 +237,7 @@ def extract_vulnerability(
|
|
|
237
237
|
outcome = extract(
|
|
238
238
|
llm, config.models[role], template, advisory_text(group, by_id), inputs, known,
|
|
239
239
|
config.prompt_budget(role), counter, config.budget.max_hunk_lines,
|
|
240
|
-
config.candidate_limit(role),
|
|
240
|
+
config.candidate_limit(role), config.validation.dotted_entry_points,
|
|
241
241
|
) # fmt: skip
|
|
242
242
|
except ResponseNotCachedError:
|
|
243
243
|
return "not_cached" # cache-only replay of a run made elsewhere: never a model call
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""SARIF 2.1.0 output, for GitHub code scanning and other SARIF readers. No LLM.
|
|
2
|
+
|
|
3
|
+
Every alert becomes one result, so nothing is hidden. The level ranks it:
|
|
4
|
+
|
|
5
|
+
* CALLED: ``error``, located at the first reference in your code;
|
|
6
|
+
* UNKNOWN: ``warning``, located at the dependency's line in the lockfile;
|
|
7
|
+
* IMPORTED_NOT_CALLED and NOT_IMPORTED: ``note``, also at the lockfile line.
|
|
8
|
+
|
|
9
|
+
Each vulnerability is one rule, identified by the alert's advisory ID. A partial fingerprint
|
|
10
|
+
over the advisory and the package keeps an alert's identity when line numbers or the pinned
|
|
11
|
+
version change, so code scanning tracks it across scans.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import posixpath
|
|
17
|
+
from collections.abc import Mapping, Sequence
|
|
18
|
+
from pathlib import Path, PurePosixPath
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from patchsight import __version__
|
|
22
|
+
from patchsight.scan.lockfiles import Dependency
|
|
23
|
+
from patchsight.scan.report import FOOTNOTE, stale_warning
|
|
24
|
+
from patchsight.scan.verdicts import Alert, Verdict, order
|
|
25
|
+
|
|
26
|
+
SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json"
|
|
27
|
+
INFORMATION_URI = "https://github.com/professor3333/PatchSight"
|
|
28
|
+
LEVELS: dict[Verdict, str] = {
|
|
29
|
+
"CALLED": "error",
|
|
30
|
+
"UNKNOWN": "warning",
|
|
31
|
+
"IMPORTED_NOT_CALLED": "note",
|
|
32
|
+
"NOT_IMPORTED": "note",
|
|
33
|
+
}
|
|
34
|
+
LABELS: dict[Verdict, str] = {
|
|
35
|
+
"CALLED": "Called",
|
|
36
|
+
"UNKNOWN": "Unknown",
|
|
37
|
+
"IMPORTED_NOT_CALLED": "Imported, not called",
|
|
38
|
+
"NOT_IMPORTED": "Not imported",
|
|
39
|
+
}
|
|
40
|
+
FINGERPRINT = "patchsight/v1"
|
|
41
|
+
MAX_RELATED = 20
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def uri(path: str, base: str = "") -> str:
|
|
45
|
+
"""A result path as a SARIF URI: relative paths stay relative, joined under ``base``."""
|
|
46
|
+
joined = posixpath.normpath(posixpath.join(base, path)) if base else posixpath.normpath(path)
|
|
47
|
+
pure = PurePosixPath(joined)
|
|
48
|
+
return Path(joined).as_uri() if pure.is_absolute() else joined
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _location(path: str, line: int, base: str = "", message: str | None = None) -> dict[str, Any]:
|
|
52
|
+
location: dict[str, Any] = {
|
|
53
|
+
"physicalLocation": {
|
|
54
|
+
"artifactLocation": {"uri": uri(path, base)},
|
|
55
|
+
"region": {"startLine": max(1, line)},
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if message:
|
|
59
|
+
location["message"] = {"text": message}
|
|
60
|
+
return location
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _message(alert: Alert) -> str:
|
|
64
|
+
also = f" (also {', '.join(alert.aliases)})" if alert.aliases else ""
|
|
65
|
+
text = (
|
|
66
|
+
f"{LABELS[alert.verdict]}: {alert.package} {alert.version} is affected by "
|
|
67
|
+
f"{alert.advisory}{also}. {alert.reason}"
|
|
68
|
+
)
|
|
69
|
+
if alert.matched:
|
|
70
|
+
text += f" Matched: {', '.join(alert.matched)}."
|
|
71
|
+
if alert.confidence:
|
|
72
|
+
text += f" Symbol data: {alert.confidence} confidence."
|
|
73
|
+
return text
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def render_sarif(
|
|
77
|
+
alerts: Sequence[Alert],
|
|
78
|
+
dependencies: Sequence[Dependency],
|
|
79
|
+
lockfile: str,
|
|
80
|
+
db_meta: Mapping[str, str],
|
|
81
|
+
code_base: str = "",
|
|
82
|
+
) -> str:
|
|
83
|
+
"""One SARIF run. ``code_base`` is the scanned directory, which code paths are relative to."""
|
|
84
|
+
ordered = order(alerts)
|
|
85
|
+
pins = {(d.name, d.version): d for d in dependencies if d.version}
|
|
86
|
+
rules: list[dict[str, Any]] = []
|
|
87
|
+
index: dict[str, int] = {}
|
|
88
|
+
results: list[dict[str, Any]] = []
|
|
89
|
+
for alert in ordered:
|
|
90
|
+
if alert.advisory not in index:
|
|
91
|
+
index[alert.advisory] = len(rules)
|
|
92
|
+
rules.append(
|
|
93
|
+
{
|
|
94
|
+
"id": alert.advisory,
|
|
95
|
+
"shortDescription": {"text": f"Vulnerable dependency: {alert.advisory}"},
|
|
96
|
+
"helpUri": f"https://osv.dev/vulnerability/{alert.advisory}",
|
|
97
|
+
"help": {
|
|
98
|
+
"text": f"{alert.advisory} affects {alert.package}. Upgrade to a fixed "
|
|
99
|
+
"version, or check whether your code reaches the vulnerable code."
|
|
100
|
+
},
|
|
101
|
+
"properties": {"tags": ["security", "dependency"]},
|
|
102
|
+
}
|
|
103
|
+
)
|
|
104
|
+
dependency = pins.get((alert.package, alert.version))
|
|
105
|
+
manifest = _location(
|
|
106
|
+
dependency.source if dependency else lockfile,
|
|
107
|
+
dependency.line if dependency else 1,
|
|
108
|
+
message=f"{alert.package}=={alert.version}",
|
|
109
|
+
)
|
|
110
|
+
if alert.verdict == "CALLED" and alert.evidence:
|
|
111
|
+
first, *rest = alert.evidence
|
|
112
|
+
primary = _location(first.path, first.line, code_base)
|
|
113
|
+
related = [
|
|
114
|
+
_location(e.path, e.line, code_base, "another reference")
|
|
115
|
+
for e in rest[: MAX_RELATED - 1]
|
|
116
|
+
] + [manifest]
|
|
117
|
+
else:
|
|
118
|
+
primary, related = manifest, []
|
|
119
|
+
result: dict[str, Any] = {
|
|
120
|
+
"ruleId": alert.advisory,
|
|
121
|
+
"ruleIndex": index[alert.advisory],
|
|
122
|
+
"level": LEVELS[alert.verdict],
|
|
123
|
+
"message": {"text": _message(alert)},
|
|
124
|
+
"locations": [primary],
|
|
125
|
+
"partialFingerprints": {
|
|
126
|
+
FINGERPRINT: hashlib.sha256(
|
|
127
|
+
f"{alert.advisory}|{alert.package}".encode()
|
|
128
|
+
).hexdigest()[:32]
|
|
129
|
+
},
|
|
130
|
+
"properties": {
|
|
131
|
+
"verdict": alert.verdict,
|
|
132
|
+
"package": alert.package,
|
|
133
|
+
"version": alert.version,
|
|
134
|
+
"aliases": list(alert.aliases),
|
|
135
|
+
"matched": list(alert.matched),
|
|
136
|
+
"confidence": alert.confidence,
|
|
137
|
+
},
|
|
138
|
+
}
|
|
139
|
+
if related:
|
|
140
|
+
for number, location in enumerate(related, 1):
|
|
141
|
+
location["id"] = number
|
|
142
|
+
result["relatedLocations"] = related
|
|
143
|
+
results.append(result)
|
|
144
|
+
notifications = [
|
|
145
|
+
{"level": "warning", "message": {"text": warning}}
|
|
146
|
+
for warning in [stale_warning(dict(db_meta))]
|
|
147
|
+
if warning
|
|
148
|
+
]
|
|
149
|
+
run: dict[str, Any] = {
|
|
150
|
+
"tool": {
|
|
151
|
+
"driver": {
|
|
152
|
+
"name": "PatchSight",
|
|
153
|
+
"semanticVersion": __version__,
|
|
154
|
+
"informationUri": INFORMATION_URI,
|
|
155
|
+
"rules": rules,
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
"invocations": [{"executionSuccessful": True, "toolExecutionNotifications": notifications}],
|
|
159
|
+
"results": results,
|
|
160
|
+
"properties": {
|
|
161
|
+
"database": {
|
|
162
|
+
k: db_meta[k] for k in ("db_version", "builder", "built_at") if k in db_meta
|
|
163
|
+
},
|
|
164
|
+
"unpinned": [
|
|
165
|
+
{
|
|
166
|
+
"name": d.raw_name,
|
|
167
|
+
"reason": d.unpinned_reason,
|
|
168
|
+
"source": d.source,
|
|
169
|
+
"line": d.line,
|
|
170
|
+
}
|
|
171
|
+
for d in dependencies
|
|
172
|
+
if not d.version
|
|
173
|
+
],
|
|
174
|
+
"note": FOOTNOTE,
|
|
175
|
+
},
|
|
176
|
+
}
|
|
177
|
+
return json.dumps({"$schema": SCHEMA, "version": "2.1.0", "runs": [run]}, indent=2)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|